From 6ee6bb55e2756e450d20f6a393896795aaecf728 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Fri, 14 Dec 2018 18:20:37 +0200 Subject: [PATCH] SQL: Introduce HISTOGRAM grouping function (#36510) Introduce Histogram grouping function for bucketing/grouping data based on a given range. Both date and numeric histograms are supported using the appropriate range declaration (numbers vs intervals). SELECT HISTOGRAM(number, 50) AS h FROM index GROUP BY h SELECT HISTOGRAM(date, INTERVAL 1 YEAR) AS h FROM index GROUP BY h In addition add multiply operator for Intervals Add docs for intervals and histogram Fix #36509 --- docs/reference/sql/concepts.asciidoc | 2 +- .../sql/functions/date-time.asciidoc | 96 ++++++++++++- .../reference/sql/functions/grouping.asciidoc | 54 ++++++++ docs/reference/sql/functions/index.asciidoc | 2 + .../sql/language/data-types.asciidoc | 73 +++++++--- .../xpack/sql/jdbc/JdbcResultSet.java | 3 +- .../xpack/sql/jdbc/TypeConverter.java | 13 +- .../sql/qa/single_node/CliExplainIT.java | 9 +- .../sql/qa/single_node/JdbcDocCsvSpecIT.java | 2 +- .../xpack/sql/qa/cli/ShowTestCase.java | 4 + .../xpack/sql/qa/jdbc/DebugSqlSpec.java | 2 +- .../sql/qa/src/main/resources/agg.csv-spec | 77 +++++++++++ .../sql/qa/src/main/resources/alias.csv-spec | 4 +- .../qa/src/main/resources/command.csv-spec | 1 + .../main/resources/datetime-interval.csv-spec | 104 ++++++++++---- .../sql/qa/src/main/resources/docs.csv-spec | 127 ++++++++++++++++++ .../sql/qa/src/main/resources/math.csv-spec | 36 ++--- .../xpack/sql/analysis/analyzer/Analyzer.java | 71 +++++++++- .../xpack/sql/analysis/analyzer/Verifier.java | 17 ++- .../xpack/sql/expression/Expressions.java | 9 ++ .../xpack/sql/expression/UnaryExpression.java | 24 +--- .../expression/function/FunctionRegistry.java | 27 +++- .../sql/expression/function/FunctionType.java | 2 + .../function/aggregate/AggregateFunction.java | 4 +- .../function/grouping/GroupingFunction.java | 95 +++++++++++++ .../grouping/GroupingFunctionAttribute.java | 54 ++++++++ .../function/grouping/Histogram.java | 84 ++++++++++++ .../datetime/DateTimeHistogramFunction.java | 2 +- .../function/scalar/datetime/Year.java | 8 +- .../sql/expression/literal/Interval.java | 5 +- .../expression/literal/IntervalDayTime.java | 5 + .../expression/literal/IntervalYearMonth.java | 7 + .../sql/expression/literal/Intervals.java | 19 +++ .../arithmetic/BinaryArithmeticProcessor.java | 23 +++- .../predicate/operator/arithmetic/Mul.java | 39 ++++++ .../xpack/sql/optimizer/Optimizer.java | 68 +--------- .../xpack/sql/parser/AbstractBuilder.java | 7 + .../xpack/sql/parser/ExpressionBuilder.java | 37 +++-- .../xpack/sql/planner/QueryFolder.java | 6 + .../xpack/sql/planner/QueryTranslator.java | 63 +++++++-- .../sql/plugin/RestSqlClearCursorAction.java | 2 +- .../xpack/sql/plugin/RestSqlQueryAction.java | 6 +- .../xpack/sql/querydsl/agg/Agg.java | 2 +- .../xpack/sql/querydsl/agg/Aggs.java | 7 +- .../sql/querydsl/agg/GroupByColumnKey.java | 36 ----- .../querydsl/agg/GroupByDateHistogram.java | 67 +++++++++ .../sql/querydsl/agg/GroupByDateKey.java | 70 ---------- .../xpack/sql/querydsl/agg/GroupByKey.java | 52 +++++-- .../querydsl/agg/GroupByNumericHistogram.java | 59 ++++++++ .../sql/querydsl/agg/GroupByScriptKey.java | 74 ---------- .../xpack/sql/querydsl/agg/GroupByValue.java | 39 ++++++ .../analyzer/VerifierErrorMessagesTests.java | 18 +++ .../search/SourceGeneratorTests.java | 6 +- .../sql/expression/TyperResolutionTests.java | 48 +++++++ .../arithmetic/BinaryArithmeticTests.java | 22 +++ .../xpack/sql/optimizer/OptimizerTests.java | 2 +- .../sql/planner/QueryTranslatorTests.java | 39 +++++- .../xpack/sql/tree/NodeSubclassTests.java | 6 + 58 files changed, 1432 insertions(+), 408 deletions(-) create mode 100644 docs/reference/sql/functions/grouping.asciidoc create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/GroupingFunction.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/GroupingFunctionAttribute.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/Histogram.java delete mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByColumnKey.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByDateHistogram.java delete mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByDateKey.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByNumericHistogram.java delete mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByScriptKey.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByValue.java create mode 100644 x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/TyperResolutionTests.java diff --git a/docs/reference/sql/concepts.asciidoc b/docs/reference/sql/concepts.asciidoc index 617091446ea..a3ea1f02e72 100644 --- a/docs/reference/sql/concepts.asciidoc +++ b/docs/reference/sql/concepts.asciidoc @@ -64,4 +64,4 @@ Multiple clusters, each with its own namespace, connected to each other in a fed |=== -As one can see while the mapping between the concepts are not exactly one to one and the semantics somewhat different, there are more things in common than differences. In fact, thanks to SQL declarative nature, many concepts can move across {es} transparently and the terminology of the two likely to be used interchangeably through-out the rest of the material. +As one can see while the mapping between the concepts are not exactly one to one and the semantics somewhat different, there are more things in common than differences. In fact, thanks to SQL declarative nature, many concepts can move across {es} transparently and the terminology of the two likely to be used interchangeably throughout the rest of the material. \ No newline at end of file diff --git a/docs/reference/sql/functions/date-time.asciidoc b/docs/reference/sql/functions/date-time.asciidoc index d3c8b844273..2ec6529f8f8 100644 --- a/docs/reference/sql/functions/date-time.asciidoc +++ b/docs/reference/sql/functions/date-time.asciidoc @@ -1,7 +1,101 @@ [role="xpack"] [testenv="basic"] [[sql-functions-datetime]] -=== Date and Time Functions +=== Date/Time and Interval Functions and Operators + +beta[] + +{es-sql} offers a wide range of facilities for performing date/time manipulations. + +[[sql-functions-datetime-interval]] +==== Intervals + +A common requirement when dealing with date/time in general revolves around +the notion of ``interval``s, a topic that is worth exploring in the context of {es} and {es-sql}. + +{es} has comprehensive support for <> both inside <> and <>. +Inside {es-sql} the former is supported as is by passing the expression in the table name, while the latter is supported through the standard SQL `INTERVAL`. + +The table below shows the mapping between {es} and {es-sql}: + +[cols="^m,^m",options="header"] + +|=== +| {es} | {es-sql} + +2+h| Index/Table date math + +2+| + +2+h| Query date math + +| 1y | INTERVAL 1 YEAR +| 2M | INTERVAL 2 MONTH +| 3w | INTERVAL 21 DAY +| 4d | INTERVAL 4 DAY +| 5h | INTERVAL 5 HOUR +| 6m | INTERVAL 6 MINUTE +| 7s | INTERVAL 7 SECOND + +|=== + +`INTERVAL` allows either `YEAR` and `MONTH` to be mixed together _or_ `DAY`, `HOUR`, `MINUTE` and `SECOND`. + +TIP: {es-sql} accepts also the plural for each time unit (e.g. both `YEAR` and `YEARS` are valid). + +Example of the possible combinations below: + +[cols="^,^",options="header"] + +|=== +| Interval | Description + +| `INTERVAL '1-2' YEAR TO MONTH` | 1 year and 2 months +| `INTERVAL '3 4' DAYS TO HOURS` | 3 days and 4 hours +| `INTERVAL '5 6:12' DAYS TO MINUTES` | 5 days, 6 hours and 12 minutes +| `INTERVAL '3 4:56:01' DAY TO SECOND` | 3 days, 4 hours, 56 minutes and 1 second +| `INTERVAL '2 3:45:01.23456789' DAY TO SECOND` | 2 days, 3 hours, 45 minutes, 1 second and 234567890 nanoseconds +| `INTERVAL '123:45' HOUR TO MINUTES` | 123 hours and 45 minutes +| `INTERVAL '65:43:21.0123' HOUR TO SECONDS` | 65 hours, 43 minutes, 21 seconds and 12300000 nanoseconds +| `INTERVAL '45:01.23' MINUTES TO SECONDS` | 45 minutes, 1 second and 230000000 nanoseconds + +|=== + +==== Operators + +Basic arithmetic operators (`+`, `-`, etc) support date-time parameters as indicated below: + +["source","sql",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{sql-specs}/docs.csv-spec[dtIntervalPlusInterval] +-------------------------------------------------- + +["source","sql",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{sql-specs}/docs.csv-spec[dtDatePlusInterval] +-------------------------------------------------- + +["source","sql",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{sql-specs}/docs.csv-spec[dtMinusInterval] +-------------------------------------------------- + +["source","sql",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{sql-specs}/docs.csv-spec[dtIntervalMinusInterval] +-------------------------------------------------- + +["source","sql",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{sql-specs}/docs.csv-spec[dtDateMinusInterval] +-------------------------------------------------- + +["source","sql",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{sql-specs}/docs.csv-spec[dtIntervalMul] +-------------------------------------------------- + +==== Functions beta[] diff --git a/docs/reference/sql/functions/grouping.asciidoc b/docs/reference/sql/functions/grouping.asciidoc new file mode 100644 index 00000000000..9a8c5c5ef53 --- /dev/null +++ b/docs/reference/sql/functions/grouping.asciidoc @@ -0,0 +1,54 @@ +[role="xpack"] +[testenv="basic"] +[[sql-functions-grouping]] +=== Grouping Functions + +beta[] + +Functions for creating special __grouping__s (also known as _bucketing_); as such these need to be used +as part of the <>. + +[[sql-functions-grouping-histogram]] +==== `HISTOGRAM` + +.Synopsis +[source, sql] +---- +HISTOGRAM ( numeric_exp<1>, numeric_interval<2>) +HISTOGRAM ( date_exp<3>, date_time_interval<4>) +---- + +*Input*: + +<1> numeric expression (typically a field) +<2> numeric interval +<3> date/time expression (typically a field) +<4> date/time <> + +*Output*: non-empty buckets or groups of the given expression divided according to the given interval + +.Description + +The histogram function takes all matching values and divides them into buckets with fixed size matching the given interval, using (roughly) the following formula: + +[source, sql] +---- +bucket_key = Math.floor(value / interval) * interval +---- + +`Histogram` can be applied on either numeric fields: + + +["source","sql",subs="attributes,callouts,macros"] +---- +include-tagged::{sql-specs}/docs.csv-spec[histogramNumeric] +---- + +or date/time fields: + +["source","sql",subs="attributes,callouts,macros"] +---- +include-tagged::{sql-specs}/docs.csv-spec[histogramDate] +---- + + diff --git a/docs/reference/sql/functions/index.asciidoc b/docs/reference/sql/functions/index.asciidoc index 552a70955cf..85c2e25f865 100644 --- a/docs/reference/sql/functions/index.asciidoc +++ b/docs/reference/sql/functions/index.asciidoc @@ -9,6 +9,7 @@ beta[] * <> * <> +* <> * <> * <> * <> @@ -19,6 +20,7 @@ beta[] include::operators.asciidoc[] include::aggs.asciidoc[] +include::grouping.asciidoc[] include::date-time.asciidoc[] include::search.asciidoc[] include::math.asciidoc[] diff --git a/docs/reference/sql/language/data-types.asciidoc b/docs/reference/sql/language/data-types.asciidoc index 0b2384de1ff..572fd094dbc 100644 --- a/docs/reference/sql/language/data-types.asciidoc +++ b/docs/reference/sql/language/data-types.asciidoc @@ -7,42 +7,71 @@ beta[] Most of {es} <> are available in {es-sql}, as indicated below: -[cols="^,^,^",options="header"] +[cols="^,^m,^",options="header"] |=== -| {es} type | SQL type | SQL precision +| {es} type | SQL type | SQL precision 3+h| Core types -| <> | `null` | 0 -| <> | `boolean` | 1 -| <> | `tinyint` | 3 -| <> | `smallint` | 5 -| <> | `integer` | 10 -| <> | `bigint` | 19 -| <> | `double` | 15 -| <> | `real` | 7 -| <> | `float` | 16 -| <> | `float` | 19 -| <> | `varchar` | based on <> -| <> | `varchar` | 2,147,483,647 -| <> | `varbinary` | 2,147,483,647 -| <> | `timestamp` | 24 - -3+h| Complex types - -| <> | `struct` | 0 -| <> | `struct` | 0 +| <> | null | 0 +| <> | boolean | 1 +| <> | tinyint | 3 +| <> | smallint | 5 +| <> | integer | 10 +| <> | bigint | 19 +| <> | double | 15 +| <> | real | 7 +| <> | float | 16 +| <> | float | 19 +| <> | varchar | based on <> +| <> | varchar | 2,147,483,647 +| <> | varbinary | 2,147,483,647 +| <> | timestamp | 24 +| <> | varchar | 39 + +3+h| Complex types + +| <> | struct | 0 +| <> | struct | 0 3+h| Unsupported types -| _types not mentioned above_ | `unsupported`| 0 +| _types not mentioned above_ | unsupported | 0 |=== + Obviously, not all types in {es} have an equivalent in SQL and vice-versa hence why, {es-sql} uses the data type _particularities_ of the former over the latter as ultimately {es} is the backing store. +In addition to the types above, {es-sql} also supports at _runtime_ SQL-specific types that do not have an equivalent in {es}. +Such types cannot be loaded from {es} (as it does not know about them) however can be used inside {es-sql} in queries or their results. + +The table below indicates these types: + +[cols="^m,^",options="header"] + +|=== +| SQL type | SQL precision + + +| interval_year | 7 +| interval_month | 7 +| interval_day | 23 +| interval_hour | 23 +| interval_minute | 23 +| interval_second | 23 +| interval_year_to_month | 7 +| interval_day_to_hour | 23 +| interval_day_to_minute | 23 +| interval_day_to_second | 23 +| interval_hour_to_minute | 23 +| interval_hour_to_second | 23 +| interval_minute_to_second | 23 + +|=== + [[sql-multi-field]] [float] diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSet.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSet.java index 62afe6c8f5f..4ef9ab4c829 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSet.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSet.java @@ -122,8 +122,7 @@ class JdbcResultSet implements ResultSet, JdbcWrapper { @Override public String getString(int columnIndex) throws SQLException { - Object val = column(columnIndex); - return val != null ? val.toString() : null; + return getObject(columnIndex, String.class); } @Override diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java index 638e31d02e5..a287d77191c 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.sql.jdbc; +import org.elasticsearch.xpack.sql.proto.StringUtils; + import java.sql.Date; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; @@ -118,10 +120,11 @@ final class TypeConverter { return (T) convert(val, columnType, typeString); } - // converting a Long to a Timestamp shouldn't be possible according to the spec, - // it feels a little brittle to check this scenario here and I don't particularly like it - // TODO: can we do any better or should we go over the spec and allow getLong(date) to be valid? - if (!(type == Long.class && columnType == EsType.DATE) && type.isInstance(val)) { + // if the value type is the same as the target, no conversion is needed + // make sure though to check the internal type against the desired one + // since otherwise the internal object format can leak out + // (for example dates when longs are requested or intervals for strings) + if (type.isInstance(val) && TypeUtils.classOf(columnType) == type) { try { return type.cast(val); } catch (ClassCastException cce) { @@ -268,7 +271,7 @@ final class TypeConverter { } private static String asString(Object nativeValue) { - return nativeValue == null ? null : String.valueOf(nativeValue); + return nativeValue == null ? null : StringUtils.toString(nativeValue); } private static T failConversion(Object value, EsType columnType, String typeString, Class target) throws SQLException { diff --git a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java index 4296c5ae069..c2027ccbfcc 100644 --- a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java +++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java @@ -26,8 +26,7 @@ public class CliExplainIT extends CliIntegrationTestCase { assertThat(command("EXPLAIN " + (randomBoolean() ? "" : "(PLAN ANALYZED) ") + "SELECT * FROM test"), containsString("plan")); assertThat(readLine(), startsWith("----------")); assertThat(readLine(), startsWith("Project[[test_field{f}#")); - assertThat(readLine(), startsWith("\\_SubQueryAlias[test]")); - assertThat(readLine(), startsWith(" \\_EsRelation[test][test_field{f}#")); + assertThat(readLine(), startsWith("\\_EsRelation[test][test_field{f}#")); assertEquals("", readLine()); assertThat(command("EXPLAIN (PLAN OPTIMIZED) SELECT * FROM test"), containsString("plan")); @@ -74,8 +73,7 @@ public class CliExplainIT extends CliIntegrationTestCase { assertThat(readLine(), startsWith("----------")); assertThat(readLine(), startsWith("Project[[i{f}#")); assertThat(readLine(), startsWith("\\_Filter[i{f}#")); - assertThat(readLine(), startsWith(" \\_SubQueryAlias[test]")); - assertThat(readLine(), startsWith(" \\_EsRelation[test][i{f}#")); + assertThat(readLine(), startsWith(" \\_EsRelation[test][i{f}#")); assertEquals("", readLine()); assertThat(command("EXPLAIN (PLAN OPTIMIZED) SELECT * FROM test WHERE i = 2"), containsString("plan")); @@ -134,8 +132,7 @@ public class CliExplainIT extends CliIntegrationTestCase { containsString("plan")); assertThat(readLine(), startsWith("----------")); assertThat(readLine(), startsWith("Aggregate[[],[COUNT(1)#")); - assertThat(readLine(), startsWith("\\_SubQueryAlias[test]")); - assertThat(readLine(), startsWith(" \\_EsRelation[test][i{f}#")); + assertThat(readLine(), startsWith("\\_EsRelation[test][i{f}#")); assertEquals("", readLine()); assertThat(command("EXPLAIN (PLAN OPTIMIZED) SELECT COUNT(*) FROM test"), containsString("plan")); diff --git a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcDocCsvSpecIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcDocCsvSpecIT.java index 7caae86da4f..f89f801d282 100644 --- a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcDocCsvSpecIT.java +++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcDocCsvSpecIT.java @@ -74,7 +74,7 @@ public class JdbcDocCsvSpecIT extends SpecBaseIntegrationTestCase { @Override protected boolean logEsResultSet() { - return true; + return false; } @Override diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ShowTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ShowTestCase.java index 8f973748981..cddbf4c1007 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ShowTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ShowTestCase.java @@ -38,6 +38,10 @@ public abstract class ShowTestCase extends CliIntegrationTestCase { while (aggregateFunction.matcher(line).matches()) { line = readLine(); } + Pattern groupingFunction = Pattern.compile("\\s*[A-Z0-9_~]+\\s*\\|\\s*GROUPING\\s*"); + while (groupingFunction.matcher(line).matches()) { + line = readLine(); + } Pattern conditionalFunction = Pattern.compile("\\s*[A-Z0-9_~]+\\s*\\|\\s*CONDITIONAL\\s*"); while (conditionalFunction.matcher(line).matches()) { line = readLine(); diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java index 21d2f3301fb..b51d66ace2e 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java @@ -16,7 +16,7 @@ public abstract class DebugSqlSpec extends SqlSpecTestCase { @ParametersFactory(shuffle = false, argumentFormatting = PARAM_FORMATTING) public static List readScriptSpec() throws Exception { Parser parser = specParser(); - return readScriptSpec("/debug.sql-spec", parser); + return readScriptSpec("/datetime.sql-spec", parser); } public DebugSqlSpec(String fileName, String groupName, String testName, Integer lineNumber, String query) { diff --git a/x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec index 3db237925d1..5d1e59ef7a2 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec @@ -137,3 +137,80 @@ null |null |null |null |null | 3 |3 |51 |3 |3.0 |100.0 |NaN |NaN 4 |4 |72 |4 |4.0 |0.0 |NaN |NaN ; + + +// +// Grouping functions +// + + +histogramNumeric +SELECT HISTOGRAM(salary, 5000) AS h FROM test_emp GROUP BY h; + + h +--------------- +25000 +30000 +35000 +40000 +45000 +50000 +55000 +60000 +65000 +70000 +; + +histogramDate +schema::h:ts|c:l +SELECT HISTOGRAM(birth_date, INTERVAL 1 YEAR) AS h, COUNT(*) as c FROM test_emp GROUP BY h; + + h | c +--------------------+--------------- +null |10 +1951-04-11T00:00:00Z|1 +1952-04-05T00:00:00Z|10 +1953-03-31T00:00:00Z|10 +1954-03-26T00:00:00Z|7 +1955-03-21T00:00:00Z|4 +1956-03-15T00:00:00Z|4 +1957-03-10T00:00:00Z|6 +1958-03-05T00:00:00Z|6 +1959-02-28T00:00:00Z|9 +1960-02-23T00:00:00Z|7 +1961-02-17T00:00:00Z|8 +1962-02-12T00:00:00Z|6 +1963-02-07T00:00:00Z|7 +1964-02-02T00:00:00Z|5 + +; + +histogramDateWithCountAndOrder +schema::h:ts|c:l +SELECT HISTOGRAM(birth_date, INTERVAL 1 YEAR) AS h, COUNT(*) as c FROM test_emp GROUP BY h ORDER BY h DESC; + + h | c +--------------------+--------------- +1964-02-02T00:00:00Z|5 +1963-02-07T00:00:00Z|7 +1962-02-12T00:00:00Z|6 +1961-02-17T00:00:00Z|8 +1960-02-23T00:00:00Z|7 +1959-02-28T00:00:00Z|9 +1958-03-05T00:00:00Z|6 +1957-03-10T00:00:00Z|6 +1956-03-15T00:00:00Z|4 +1955-03-21T00:00:00Z|4 +1954-03-26T00:00:00Z|7 +1953-03-31T00:00:00Z|10 +1952-04-05T00:00:00Z|10 +1951-04-11T00:00:00Z|1 +null |10 +; + +histogramDateWithDateFunction-Ignore +SELECT YEAR(HISTOGRAM(birth_date, INTERVAL 1 YEAR)) AS h, COUNT(*) as c FROM test_emp GROUP BY h ORDER BY h DESC; + + + +; \ No newline at end of file diff --git a/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec index 704623b5eec..e87aaecf6f3 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec @@ -92,7 +92,7 @@ test_emp | BASE TABLE test_emp_copy | BASE TABLE ; -testGroupByOnAlias +groupByOnAlias SELECT gender g, PERCENTILE(emp_no, 97) p1 FROM test_alias GROUP BY g ORDER BY g DESC; g:s | p1:d @@ -102,7 +102,7 @@ F | 10099.52 null | 10019.0 ; -testGroupByOnPattern +groupByOnPattern SELECT gender, PERCENTILE(emp_no, 97) p1 FROM "test_*" WHERE gender is NOT NULL GROUP BY gender; gender:s | p1:d diff --git a/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec index 9c2b664680b..7c9c98f6d04 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec @@ -19,6 +19,7 @@ SKEWNESS |AGGREGATE STDDEV_POP |AGGREGATE SUM_OF_SQUARES |AGGREGATE VAR_POP |AGGREGATE +HISTOGRAM |GROUPING COALESCE |CONDITIONAL GREATEST |CONDITIONAL IFNULL |CONDITIONAL diff --git a/x-pack/plugin/sql/qa/src/main/resources/datetime-interval.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/datetime-interval.csv-spec index c20529485da..9434ead51da 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/datetime-interval.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/datetime-interval.csv-spec @@ -3,7 +3,7 @@ // hence why all INTERVAL tests need to be done manually // -testExactIntervals +exactIntervals SELECT INTERVAL 1 YEAR AS y, INTERVAL 2 MONTH AS m, INTERVAL 3 DAY AS d, INTERVAL 4 HOUR AS h, INTERVAL 5 MINUTE AS mm, INTERVAL 6 SECOND AS s; y | m | d | h | mm | s @@ -20,7 +20,7 @@ SELECT INTERVAL 1 YEARS AS y, INTERVAL 2 MONTHS AS m, INTERVAL 3 DAYS AS d, INTE ; // take the examples from https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/interval-literals?view=sql-server-2017 -testYear +year SELECT INTERVAL '326' YEAR; INTERVAL '326' YEAR @@ -28,7 +28,7 @@ INTERVAL '326' YEAR +326-0 ; -testMonth +month SELECT INTERVAL '326' MONTH; INTERVAL '326' MONTH @@ -36,7 +36,7 @@ INTERVAL '326' MONTH +0-326 ; -testDay +day SELECT INTERVAL '3261' DAY; INTERVAL '3261' DAY @@ -44,7 +44,7 @@ INTERVAL '3261' DAY +3261 00:00:00.0 ; -testHour +hour SELECT INTERVAL '163' HOUR; INTERVAL '163' HOUR @@ -52,7 +52,7 @@ INTERVAL '163' HOUR +6 19:00:00.0 ; -testMinute +minute SELECT INTERVAL '163' MINUTE; INTERVAL '163' MINUTE @@ -60,7 +60,7 @@ INTERVAL '163' MINUTE +0 02:43:00.0 ; -testSecond +second SELECT INTERVAL '223.16' SECOND; INTERVAL '223.16' SECOND @@ -68,7 +68,7 @@ INTERVAL '223.16' SECOND +0 00:03:43.16 ; -testYearMonth +yearMonth SELECT INTERVAL '163-11' YEAR TO MONTH; INTERVAL '163-11' YEAR TO MONTH @@ -76,7 +76,7 @@ INTERVAL '163-11' YEAR TO MONTH +163-11 ; -testDayHour +dayHour SELECT INTERVAL '163 12' DAY TO HOUR; INTERVAL '163 12' DAY TO HOUR @@ -84,7 +84,7 @@ INTERVAL '163 12' DAY TO HOUR +163 12:00:00.0 ; -testDayMinute +dayMinute SELECT INTERVAL '163 12:39' DAY TO MINUTE AS interval; interval @@ -92,7 +92,7 @@ interval +163 12:39:00.0 ; -testDaySecond +daySecond SELECT INTERVAL '163 12:39:59.163' DAY TO SECOND AS interval; interval @@ -100,7 +100,7 @@ interval +163 12:39:59.163 ; -testDaySecondNegative +daySecondNegative SELECT INTERVAL -'163 23:39:56.23' DAY TO SECOND AS interval; interval @@ -108,7 +108,7 @@ interval -163 23:39:56.23 ; -testHourMinute +hourMinute SELECT INTERVAL '163:39' HOUR TO MINUTE AS interval; interval @@ -116,7 +116,7 @@ interval +6 19:39:00.0 ; -testHourSecond +hourSecond SELECT INTERVAL '163:39:59.163' HOUR TO SECOND AS interval; interval @@ -124,7 +124,7 @@ interval +6 19:39:59.163 ; -testMinuteSecond +minuteSecond SELECT INTERVAL '163:59.163' MINUTE TO SECOND AS interval; interval @@ -132,7 +132,65 @@ interval +0 02:43:59.163 ; -testDatePlusInterval +intervalPlusInterval +SELECT INTERVAL 1 DAY + INTERVAL 53 MINUTES; + +INTERVAL 1 DAY + INTERVAL 53 MINUTES +------------------------------------ ++1 00:53:00.0 +; + +datePlusIntervalInline +SELECT CAST('1969-05-13T12:34:56' AS DATE) + INTERVAL 49 YEARS AS result; + + result +-------------------- +2018-05-13T12:34:56Z +; + +minusInterval +SELECT - INTERVAL '49-1' YEAR TO MONTH result; + + result +--------------- +-49-1 +; + + +intervalMinusInterval +SELECT INTERVAL '1' DAY - INTERVAL '2' HOURS AS result; + + result +--------------- ++0 22:00:00.0 +; + + +intervalYearMultiply +SELECT -2 * INTERVAL '3' YEARS AS result; + + result +--------------- +-6-0 +; + +intervalDayMultiply +SELECT -2 * INTERVAL '1 23:45' DAY TO MINUTES AS result; + + result +--------------- +-3 23:30:00.0 +; + +dateMinusInterval +SELECT CAST('2018-05-13T12:34:56' AS DATE) - INTERVAL '2-8' YEAR TO MONTH AS result; + + result +-------------------- +2015-09-13T12:34:56Z +; + +datePlusInterval SELECT MONTH(birth_date) AS m, MONTH(birth_date + INTERVAL '1-2' YEAR TO MONTH) AS f FROM test_emp GROUP BY birth_date ORDER BY birth_date ASC LIMIT 5; m | f @@ -144,7 +202,7 @@ null |null 6 |8 ; -testDatePlusMixInterval +datePlusMixInterval SELECT birth_date, birth_date + INTERVAL '1-2' YEAR TO MONTH AS f FROM test_emp GROUP BY birth_date ORDER BY birth_date ASC LIMIT 5; birth_date:ts | f:ts @@ -157,7 +215,7 @@ null |null ; -testDateMinusInterval +dateMinusInterval SELECT YEAR(birth_date) AS y, YEAR(birth_date - INTERVAL 1 YEAR) AS f FROM test_emp GROUP BY birth_date ORDER BY birth_date ASC LIMIT 5; y | f @@ -169,7 +227,7 @@ null |null 1952 |1951 ; -testDatePlusMixInterval +datePlusMixInterval SELECT birth_date, birth_date + INTERVAL '1-2' YEAR TO MONTH AS f FROM test_emp GROUP BY birth_date ORDER BY birth_date ASC LIMIT 5; birth_date:ts | f:ts @@ -182,7 +240,7 @@ null |null ; -testDateAndMultipleIntervals +dateAndMultipleIntervals SELECT birth_date, birth_date - INTERVAL 1 YEAR + INTERVAL '2-3' YEAR TO MONTH AS f FROM test_emp GROUP BY birth_date ORDER BY birth_date ASC LIMIT 5; birth_date:ts | f:ts @@ -195,7 +253,7 @@ null |null ; -testDatePlusIntervalWhereClause +datePlusIntervalWhereClause SELECT birth_date, YEAR(birth_date + INTERVAL 1 YEAR) AS f FROM test_emp WHERE YEAR(birth_date + INTERVAL 1 YEAR) > 1 GROUP BY birth_date ORDER BY birth_date ASC LIMIT 5; birth_date:ts | f:i @@ -207,7 +265,7 @@ SELECT birth_date, YEAR(birth_date + INTERVAL 1 YEAR) AS f FROM test_emp WHERE Y 1952-07-08T00:00:00Z|1953 ; -testDateMinusIntervalOrder +dateMinusIntervalOrder SELECT birth_date, MONTH(birth_date - INTERVAL 1 YEAR) AS f FROM test_emp GROUP BY birth_date ORDER BY MONTH(birth_date - INTERVAL 1 YEAR) ASC LIMIT 5; birth_date:ts | f:i @@ -220,7 +278,7 @@ null |null ; // see https://github.com/elastic/elasticsearch/issues/35745 -testDatePlusIntervalHavingClause-Ignore +datePlusIntervalHavingClause-Ignore SELECT birth_date, MAX(hire_date) - INTERVAL 1 YEAR AS f FROM test_emp GROUP BY birth_date ORDER BY birth_date ASC LIMIT 5; birth_date:ts | f:ts diff --git a/x-pack/plugin/sql/qa/src/main/resources/docs.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/docs.csv-spec index fe0d3eba3b0..03d412b2ab5 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/docs.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/docs.csv-spec @@ -196,6 +196,7 @@ SKEWNESS |AGGREGATE STDDEV_POP |AGGREGATE SUM_OF_SQUARES |AGGREGATE VAR_POP |AGGREGATE +HISTOGRAM |GROUPING COALESCE |CONDITIONAL GREATEST |CONDITIONAL IFNULL |CONDITIONAL @@ -299,6 +300,7 @@ CONVERT |SCALAR DATABASE |SCALAR USER |SCALAR SCORE |SCORE + // end::showFunctions ; @@ -697,6 +699,131 @@ SELECT MIN(salary) AS min, MAX(salary) AS max FROM emp HAVING min > 25000; // end::groupByHavingImplicitNoMatch //; +/////////////////////////////// +// +// Grouping +// +/////////////////////////////// + +histogramNumeric +// tag::histogramNumeric +SELECT HISTOGRAM(salary, 5000) AS h FROM emp GROUP BY h; + + h +--------------- +25000 +30000 +35000 +40000 +45000 +50000 +55000 +60000 +65000 +70000 + +// end::histogramNumeric +; + +histogramDate +schema::h:ts|c:l +// tag::histogramDate +SELECT HISTOGRAM(birth_date, INTERVAL 1 YEAR) AS h, COUNT(*) AS c FROM emp GROUP BY h; + + + h | c +--------------------+--------------- +null |10 +1951-04-11T00:00:00Z|1 +1952-04-05T00:00:00Z|10 +1953-03-31T00:00:00Z|10 +1954-03-26T00:00:00Z|7 +1955-03-21T00:00:00Z|4 +1956-03-15T00:00:00Z|4 +1957-03-10T00:00:00Z|6 +1958-03-05T00:00:00Z|6 +1959-02-28T00:00:00Z|9 +1960-02-23T00:00:00Z|7 +1961-02-17T00:00:00Z|8 +1962-02-12T00:00:00Z|6 +1963-02-07T00:00:00Z|7 +1964-02-02T00:00:00Z|5 + +// end::histogramDate +; + +/////////////////////////////// +// +// Date/Time +// +/////////////////////////////// + + +dtIntervalPlusInterval +// tag::dtIntervalPlusInterval +SELECT INTERVAL 1 DAY + INTERVAL 53 MINUTES AS result; + + result +--------------- ++1 00:53:00.0 + +// end::dtIntervalPlusInterval +; + + +dtDatePlusInterval +// tag::dtDatePlusInterval +SELECT CAST('1969-05-13T12:34:56' AS DATE) + INTERVAL 49 YEARS AS result; + + result +-------------------- +2018-05-13T12:34:56Z +// end::dtDatePlusInterval +; + +dtMinusInterval +// tag::dtMinusInterval +SELECT - INTERVAL '49-1' YEAR TO MONTH result; + + result +--------------- +-49-1 + +// end::dtMinusInterval +; + +dtIntervalMinusInterval +// tag::dtIntervalMinusInterval +SELECT INTERVAL '1' DAY - INTERVAL '2' HOURS AS result; + + result +--------------- ++0 22:00:00.0 +// end::dtIntervalMinusInterval +; + + +dtDateMinusInterval +// tag::dtDateMinusInterval +SELECT CAST('2018-05-13T12:34:56' AS DATE) - INTERVAL '2-8' YEAR TO MONTH AS result; + + result +-------------------- +2015-09-13T12:34:56Z +// end::dtDateMinusInterval +; + +dtIntervalMul +// tag::dtIntervalMul +SELECT -2 * INTERVAL '3' YEARS AS result; + + result +--------------- +-6-0 +// end::dtIntervalMul +; + + /////////////////////////////// // // Order by diff --git a/x-pack/plugin/sql/qa/src/main/resources/math.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/math.csv-spec index 7a63f412f43..2df93b37954 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/math.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/math.csv-spec @@ -99,13 +99,13 @@ SELECT MIN(salary) mi, MAX(salary) ma, YEAR(hire_date) year, ROUND(AVG(languages mi:i | ma:i | year:i |ROUND(AVG(languages),1):d|TRUNCATE(AVG(languages),1):d| COUNT(1):l ---------------+---------------+---------------+-------------------------+----------------------------+--------------- -25324 |70011 |1987 |3.0 |3.0 |15 -25945 |73578 |1988 |2.9 |2.8 |9 -25976 |74970 |1989 |3.0 |3.0 |13 -31120 |71165 |1990 |3.1 |3.0 |12 -30404 |58715 |1993 |3.0 |3.0 |3 -35742 |67492 |1994 |2.8 |2.7 |4 -45656 |45656 |1996 |3.0 |3.0 |1 +25324 |70011 |1986 |3.0 |3.0 |15 +25945 |73578 |1987 |2.9 |2.8 |9 +25976 |74970 |1988 |3.0 |3.0 |13 +31120 |71165 |1989 |3.1 |3.0 |12 +30404 |58715 |1992 |3.0 |3.0 |3 +35742 |67492 |1993 |2.8 |2.7 |4 +45656 |45656 |1995 |3.0 |3.0 |1 ; minMaxRoundWithHavingRound @@ -113,17 +113,17 @@ SELECT MIN(salary) mi, MAX(salary) ma, YEAR(hire_date) year, ROUND(AVG(languages mi:i | ma:i | year:i |ROUND(AVG(languages),1):d| COUNT(1):l ---------------+---------------+---------------+-------------------------+--------------- -26436 |74999 |1985 |3.1 |11 -31897 |61805 |1986 |3.5 |11 -25324 |70011 |1987 |3.0 |15 -25945 |73578 |1988 |2.9 |9 -25976 |74970 |1989 |3.0 |13 -31120 |71165 |1990 |3.1 |12 -32568 |65030 |1991 |3.3 |6 -27215 |60781 |1992 |4.1 |8 -30404 |58715 |1993 |3.0 |3 -35742 |67492 |1994 |2.8 |4 -45656 |45656 |1996 |3.0 |1 +26436 |74999 |1984 |3.1 |11 +31897 |61805 |1985 |3.5 |11 +25324 |70011 |1986 |3.0 |15 +25945 |73578 |1987 |2.9 |9 +25976 |74970 |1988 |3.0 |13 +31120 |71165 |1989 |3.1 |12 +32568 |65030 |1990 |3.3 |6 +27215 |60781 |1991 |4.1 |8 +30404 |58715 |1992 |3.0 |3 +35742 |67492 |1993 |2.8 |4 +45656 |45656 |1995 |3.0 |1 ; groupByAndOrderByTruncateWithPositiveParameter diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java index 3d407bef286..a59ad1155f1 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java @@ -108,7 +108,11 @@ public class Analyzer extends RuleExecutor { new ResolveAggsInHaving() //new ImplicitCasting() ); - return Arrays.asList(substitution, resolution); + Batch finish = new Batch("Finish Analysis", + new PruneSubqueryAliases(), + CleanAliases.INSTANCE + ); + return Arrays.asList(substitution, resolution, finish); } public LogicalPlan analyze(LogicalPlan plan) { @@ -1057,6 +1061,69 @@ public class Analyzer extends RuleExecutor { } } + + public static class PruneSubqueryAliases extends AnalyzeRule { + + @Override + protected LogicalPlan rule(SubQueryAlias alias) { + return alias.child(); + } + + @Override + protected boolean skipResolved() { + return false; + } + } + + public static class CleanAliases extends AnalyzeRule { + + public static final CleanAliases INSTANCE = new CleanAliases(); + + @Override + protected LogicalPlan rule(LogicalPlan plan) { + if (plan instanceof Project) { + Project p = (Project) plan; + return new Project(p.location(), p.child(), cleanExpressions(p.projections())); + } + + if (plan instanceof Aggregate) { + Aggregate a = (Aggregate) plan; + // clean group expressions + List cleanedGroups = a.groupings().stream().map(CleanAliases::trimAliases).collect(toList()); + return new Aggregate(a.location(), a.child(), cleanedGroups, cleanExpressions(a.aggregates())); + } + + return plan.transformExpressionsOnly(e -> { + if (e instanceof Alias) { + return ((Alias) e).child(); + } + return e; + }); + } + + private List cleanExpressions(List args) { + return args.stream().map(CleanAliases::trimNonTopLevelAliases).map(NamedExpression.class::cast).collect(toList()); + } + + public static Expression trimNonTopLevelAliases(Expression e) { + if (e instanceof Alias) { + Alias a = (Alias) e; + return new Alias(a.location(), a.name(), a.qualifier(), trimAliases(a.child()), a.id()); + } + return trimAliases(e); + } + + private static Expression trimAliases(Expression e) { + return e.transformDown(Alias::child, Alias.class); + } + + @Override + protected boolean skipResolved() { + return false; + } + } + + abstract static class AnalyzeRule extends Rule { // transformUp (post-order) - that is first children and then the node @@ -1073,4 +1140,4 @@ public class Analyzer extends RuleExecutor { return true; } } -} +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index 3fb0e7721f4..386be1eadcc 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -462,7 +462,7 @@ public final class Verifier { Map> missing = new LinkedHashMap<>(); a.aggregates().forEach(ne -> - ne.collectFirstChildren(c -> checkGroupMatch(c, ne, a.groupings(), missing, functions))); + ne.collectFirstChildren(c -> checkGroupMatch(c, ne, a.groupings(), missing, functions))); if (!missing.isEmpty()) { String plural = missing.size() > 1 ? "s" : StringUtils.EMPTY; @@ -478,6 +478,13 @@ public final class Verifier { private static boolean checkGroupMatch(Expression e, Node source, List groupings, Map> missing, Map functions) { + + // 1:1 match + if (Expressions.match(groupings, e::semanticEquals)) { + return true; + } + + // resolve FunctionAttribute to backing functions if (e instanceof FunctionAttribute) { FunctionAttribute fa = (FunctionAttribute) e; @@ -521,12 +528,14 @@ public final class Verifier { if (Functions.isAggregate(e)) { return true; } + // left without leaves which have to match; if not there's a failure - + // make sure to match directly on the expression and not on the tree + // (since otherwise exp might match the function argument which would be incorrect) final Expression exp = e; if (e.children().isEmpty()) { - if (!Expressions.anyMatch(groupings, c -> exp.semanticEquals(exp instanceof Attribute ? Expressions.attribute(c) : c))) { - missing.put(e, source); + if (Expressions.match(groupings, c -> exp.semanticEquals(exp instanceof Attribute ? Expressions.attribute(c) : c)) == false) { + missing.put(exp, source); } return true; } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java index aa6a2b0e89a..58b559d93a2 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java @@ -70,6 +70,15 @@ public final class Expressions { return false; } + public static boolean match(List exps, Predicate predicate) { + for (Expression exp : exps) { + if (predicate.test(exp)) { + return true; + } + } + return false; + } + public static boolean nullable(List exps) { for (Expression exp : exps) { if (exp.nullable()) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/UnaryExpression.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/UnaryExpression.java index c2e764522f4..1dab263f0a4 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/UnaryExpression.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/UnaryExpression.java @@ -5,9 +5,6 @@ */ package org.elasticsearch.xpack.sql.expression; -import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; -import org.elasticsearch.xpack.sql.expression.gen.pipeline.Pipe; -import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.type.DataType; @@ -16,12 +13,12 @@ import java.util.Objects; import static java.util.Collections.singletonList; -public abstract class UnaryExpression extends NamedExpression { +public abstract class UnaryExpression extends Expression { private final Expression child; protected UnaryExpression(Location location, Expression child) { - super(location, null, singletonList(child), null); + super(location, singletonList(child)); this.child = child; } @@ -58,21 +55,6 @@ public abstract class UnaryExpression extends NamedExpression { return child.dataType(); } - @Override - public Attribute toAttribute() { - throw new SqlIllegalArgumentException("Not supported yet"); - } - - @Override - public ScriptTemplate asScript() { - throw new SqlIllegalArgumentException("Not supported yet"); - } - - @Override - protected Pipe makePipe() { - throw new SqlIllegalArgumentException("Not supported yet"); - } - @Override public int hashCode() { return Objects.hash(child); @@ -91,4 +73,4 @@ public abstract class UnaryExpression extends NamedExpression { UnaryExpression other = (UnaryExpression) obj; return Objects.equals(child, other.child); } -} +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java index 9915571b83f..00581ffd84e 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.sql.expression.function.aggregate.StddevPop; import org.elasticsearch.xpack.sql.expression.function.aggregate.Sum; import org.elasticsearch.xpack.sql.expression.function.aggregate.SumOfSquares; import org.elasticsearch.xpack.sql.expression.function.aggregate.VarPop; +import org.elasticsearch.xpack.sql.expression.function.grouping.Histogram; import org.elasticsearch.xpack.sql.expression.function.scalar.Cast; import org.elasticsearch.xpack.sql.expression.function.scalar.Database; import org.elasticsearch.xpack.sql.expression.function.scalar.User; @@ -153,6 +154,8 @@ public class FunctionRegistry { def(SumOfSquares.class, SumOfSquares::new, "SUM_OF_SQUARES"), def(Skewness.class, Skewness::new, "SKEWNESS"), def(Kurtosis.class, Kurtosis::new, "KURTOSIS")); + // histogram + addToMap(def(Histogram.class, Histogram::new, "HISTOGRAM")); // Scalar functions // Conditional addToMap(def(Coalesce.class, Coalesce::new, "COALESCE"), @@ -447,6 +450,28 @@ public class FunctionRegistry { T build(Location location, Expression target, TimeZone tz); } + /** + * Build a {@linkplain FunctionDefinition} for a binary function that + * requires a timezone. + */ + @SuppressWarnings("overloads") // These are ambiguous if you aren't using ctor references but we always do + static FunctionDefinition def(Class function, DatetimeBinaryFunctionBuilder ctorRef, String... names) { + FunctionBuilder builder = (location, children, distinct, cfg) -> { + if (children.size() != 2) { + throw new IllegalArgumentException("expects exactly two arguments"); + } + if (distinct) { + throw new IllegalArgumentException("does not support DISTINCT yet it was specified"); + } + return ctorRef.build(location, children.get(0), children.get(1), cfg.timeZone()); + }; + return def(function, builder, false, names); + } + + interface DatetimeBinaryFunctionBuilder { + T build(Location location, Expression lhs, Expression rhs, TimeZone tz); + } + /** * Build a {@linkplain FunctionDefinition} for a binary function that is * not aware of time zone and does not support {@code DISTINCT}. @@ -559,4 +584,4 @@ public class FunctionRegistry { private interface CastFunctionBuilder { T build(Location location, Expression expression, DataType dataType); } -} +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionType.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionType.java index b2f4ab8ef2c..22b6a50d8ee 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionType.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionType.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.sql.expression.function; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.sql.expression.function.grouping.GroupingFunction; import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.sql.expression.predicate.conditional.ConditionalFunction; @@ -15,6 +16,7 @@ public enum FunctionType { AGGREGATE(AggregateFunction.class), CONDITIONAL(ConditionalFunction.class), + GROUPING(GroupingFunction.class), SCALAR(ScalarFunction.class), SCORE(Score.class); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/AggregateFunction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/AggregateFunction.java index 2b558970df5..e85824beaff 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/AggregateFunction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/AggregateFunction.java @@ -30,11 +30,11 @@ public abstract class AggregateFunction extends Function { private AggregateFunctionAttribute lazyAttribute; - AggregateFunction(Location location, Expression field) { + protected AggregateFunction(Location location, Expression field) { this(location, field, emptyList()); } - AggregateFunction(Location location, Expression field, List parameters) { + protected AggregateFunction(Location location, Expression field, List parameters) { super(location, CollectionUtils.combine(singletonList(field), parameters)); this.field = field; this.parameters = parameters; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/GroupingFunction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/GroupingFunction.java new file mode 100644 index 00000000000..dbfef6aeb5d --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/GroupingFunction.java @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.grouping; + +import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.Function; +import org.elasticsearch.xpack.sql.expression.gen.pipeline.AggNameInput; +import org.elasticsearch.xpack.sql.expression.gen.pipeline.Pipe; +import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.util.CollectionUtils; + +import java.util.List; +import java.util.Objects; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; + +/** + * A type of {@code Function} that creates groups or buckets. + */ +public abstract class GroupingFunction extends Function { + + private final Expression field; + private final List parameters; + + private GroupingFunctionAttribute lazyAttribute; + + protected GroupingFunction(Location location, Expression field) { + this(location, field, emptyList()); + } + + protected GroupingFunction(Location location, Expression field, List parameters) { + super(location, CollectionUtils.combine(singletonList(field), parameters)); + this.field = field; + this.parameters = parameters; + } + + public Expression field() { + return field; + } + + public List parameters() { + return parameters; + } + + @Override + public GroupingFunctionAttribute toAttribute() { + if (lazyAttribute == null) { + // this is highly correlated with QueryFolder$FoldAggregate#addFunction (regarding the function name within the querydsl) + lazyAttribute = new GroupingFunctionAttribute(location(), name(), dataType(), id(), functionId()); + } + return lazyAttribute; + } + + @Override + public final GroupingFunction replaceChildren(List newChildren) { + if (newChildren.size() != 1) { + throw new IllegalArgumentException("expected [1] child but received [" + newChildren.size() + "]"); + } + return replaceChild(newChildren.get(0)); + } + + protected abstract GroupingFunction replaceChild(Expression newChild); + + @Override + protected Pipe makePipe() { + // unresolved AggNameInput (should always get replaced by the folder) + return new AggNameInput(location(), this, name()); + } + + @Override + public ScriptTemplate asScript() { + throw new SqlIllegalArgumentException("Grouping functions cannot be scripted"); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + GroupingFunction other = (GroupingFunction) obj; + return Objects.equals(other.field(), field()) + && Objects.equals(other.parameters(), parameters()); + } + + @Override + public int hashCode() { + return Objects.hash(field(), parameters()); + } +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/GroupingFunctionAttribute.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/GroupingFunctionAttribute.java new file mode 100644 index 00000000000..4deac8a2f9e --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/GroupingFunctionAttribute.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.grouping; + +import org.elasticsearch.xpack.sql.expression.Attribute; +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.ExpressionId; +import org.elasticsearch.xpack.sql.expression.function.FunctionAttribute; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.type.DataType; + +public class GroupingFunctionAttribute extends FunctionAttribute { + + GroupingFunctionAttribute(Location location, String name, DataType dataType, ExpressionId id, String functionId) { + this(location, name, dataType, null, false, id, false, functionId); + } + + public GroupingFunctionAttribute(Location location, String name, DataType dataType, String qualifier, + boolean nullable, ExpressionId id, boolean synthetic, String functionId) { + super(location, name, dataType, qualifier, nullable, id, synthetic, functionId); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, GroupingFunctionAttribute::new, + name(), dataType(), qualifier(), nullable(), id(), synthetic(), functionId()); + } + + @Override + protected Expression canonicalize() { + return new GroupingFunctionAttribute(location(), "", dataType(), null, true, id(), false, ""); + } + + @Override + protected Attribute clone(Location location, String name, String qualifier, boolean nullable, ExpressionId id, boolean synthetic) { + // this is highly correlated with QueryFolder$FoldAggregate#addFunction (regarding the function name within the querydsl) + // that is the functionId is actually derived from the expression id to easily track it across contexts + return new GroupingFunctionAttribute(location, name, dataType(), qualifier, nullable, id, synthetic, functionId()); + } + + public GroupingFunctionAttribute withFunctionId(String functionId, String propertyPath) { + return new GroupingFunctionAttribute(location(), name(), dataType(), qualifier(), nullable(), + id(), synthetic(), functionId); + } + + @Override + protected String label() { + return "g->" + functionId(); + } +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/Histogram.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/Histogram.java new file mode 100644 index 00000000000..200682d980a --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/Histogram.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.expression.function.grouping; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.Expressions; +import org.elasticsearch.xpack.sql.expression.Expressions.ParamOrdinal; +import org.elasticsearch.xpack.sql.expression.Literal; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DataTypes; + +import java.util.Objects; +import java.util.TimeZone; + +public class Histogram extends GroupingFunction { + + private final Literal interval; + private final TimeZone timeZone; + + public Histogram(Location location, Expression field, Expression interval, TimeZone timeZone) { + super(location, field); + this.interval = (Literal) interval; + this.timeZone = timeZone; + } + + public Literal interval() { + return interval; + } + + public TimeZone timeZone() { + return timeZone; + } + + @Override + protected TypeResolution resolveType() { + TypeResolution resolution = Expressions.typeMustBeNumericOrDate(field(), "HISTOGRAM", ParamOrdinal.FIRST); + if (resolution == TypeResolution.TYPE_RESOLVED) { + // interval must be Literal interval + if (field().dataType() == DataType.DATE) { + resolution = Expressions.typeMustBe(interval, DataTypes::isInterval, "(Date) HISTOGRAM", ParamOrdinal.SECOND, "interval"); + } else { + resolution = Expressions.typeMustBeNumeric(interval, "(Numeric) HISTOGRAM", ParamOrdinal.SECOND); + } + } + + return resolution; + } + + @Override + protected GroupingFunction replaceChild(Expression newChild) { + return new Histogram(location(), newChild, interval, timeZone); + } + + @Override + public DataType dataType() { + return field().dataType(); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Histogram::new, field(), interval, timeZone); + } + + @Override + public int hashCode() { + return Objects.hash(field(), interval, timeZone); + } + + @Override + public boolean equals(Object obj) { + if (super.equals(obj)) { + Histogram other = (Histogram) obj; + return Objects.equals(interval, other.interval) + && Objects.equals(timeZone, other.timeZone); + } + return false; + } +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeHistogramFunction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeHistogramFunction.java index 60d39e7ea30..1a60ba66f48 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeHistogramFunction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeHistogramFunction.java @@ -24,5 +24,5 @@ public abstract class DateTimeHistogramFunction extends DateTimeFunction { /** * used for aggregration (date histogram) */ - public abstract String interval(); + public abstract long interval(); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/Year.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/Year.java index 2eb08c7dd93..0ba4c47058d 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/Year.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/Year.java @@ -11,11 +11,15 @@ import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.tree.NodeInfo.NodeCtor2; import java.util.TimeZone; +import java.util.concurrent.TimeUnit; /** * Extract the year from a datetime. */ public class Year extends DateTimeHistogramFunction { + + private static long YEAR_IN_MILLIS = TimeUnit.DAYS.toMillis(1) * 365L; + public Year(Location location, Expression field, TimeZone timeZone) { super(location, field, timeZone, DateTimeExtractor.YEAR); } @@ -41,7 +45,7 @@ public class Year extends DateTimeHistogramFunction { } @Override - public String interval() { - return "year"; + public long interval() { + return YEAR_IN_MILLIS; } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/literal/Interval.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/literal/Interval.java index 0aabf3b76b6..fe83b9b0099 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/literal/Interval.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/literal/Interval.java @@ -9,6 +9,7 @@ package org.elasticsearch.xpack.sql.expression.literal; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.sql.proto.StringUtils; import org.elasticsearch.xpack.sql.type.DataType; import java.io.IOException; @@ -46,6 +47,8 @@ public abstract class Interval implements NamedWriteab public abstract Interval sub(Interval interval); + public abstract Interval mul(long mul); + @Override public int hashCode() { return Objects.hash(interval, intervalType); @@ -73,6 +76,6 @@ public abstract class Interval implements NamedWriteab @Override public String toString() { - return intervalType.name() + "[" + interval + "]"; + return StringUtils.toString(interval); } } \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/literal/IntervalDayTime.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/literal/IntervalDayTime.java index 7fa0e1696ec..4f370bc313f 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/literal/IntervalDayTime.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/literal/IntervalDayTime.java @@ -54,4 +54,9 @@ public class IntervalDayTime extends Interval { public IntervalDayTime sub(Interval interval) { return new IntervalDayTime(interval().minus(interval.interval()), DataTypes.compatibleInterval(dataType(), interval.dataType())); } + + @Override + public Interval mul(long mul) { + return new IntervalDayTime(interval().multipliedBy(mul), dataType()); + } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/literal/IntervalYearMonth.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/literal/IntervalYearMonth.java index f4267f3716d..cb4346428ae 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/literal/IntervalYearMonth.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/literal/IntervalYearMonth.java @@ -9,6 +9,7 @@ package org.elasticsearch.xpack.sql.expression.literal; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DataTypeConversion; import org.elasticsearch.xpack.sql.type.DataTypes; import java.io.IOException; @@ -58,4 +59,10 @@ public class IntervalYearMonth extends Interval { return new IntervalYearMonth(interval().minus(interval.interval()).normalized(), DataTypes.compatibleInterval(dataType(), interval.dataType())); } + + @Override + public Interval mul(long mul) { + int i = DataTypeConversion.safeToInt(mul); + return new IntervalYearMonth(interval().multipliedBy(i), dataType()); + } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/literal/Intervals.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/literal/Intervals.java index a64535e83b7..0d194be105f 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/literal/Intervals.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/literal/Intervals.java @@ -10,9 +10,12 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; +import org.elasticsearch.xpack.sql.expression.Foldables; +import org.elasticsearch.xpack.sql.expression.Literal; import org.elasticsearch.xpack.sql.parser.ParsingException; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.util.Check; import org.elasticsearch.xpack.sql.util.StringUtils; import java.time.Duration; @@ -44,6 +47,22 @@ public final class Intervals { private Intervals() {} + public static long inMillis(Literal literal) { + Object fold = Foldables.valueOf(literal); + Check.isTrue(fold instanceof Interval, "Expected interval, received [{}]", fold); + TemporalAmount interval = ((Interval) fold).interval(); + long millis = 0; + if (interval instanceof Period) { + Period p = (Period) interval; + millis = p.toTotalMonths() * 30 * 24 * 60 * 60 * 1000; + } else { + Duration d = (Duration) interval; + millis = d.toMillis(); + } + + return millis; + } + public static TemporalAmount of(Location source, long duration, TimeUnit unit) { // Cannot use Period.of since it accepts int so use plus which accepts long // Further more Period and Duration have inconsistent addition methods but plus is there diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessor.java index aa0c3e830b9..1b7afa20307 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessor.java @@ -82,7 +82,28 @@ public class BinaryArithmeticProcessor extends FunctionalBinaryProcessor { + if (l instanceof Number && r instanceof Number) { + return Arithmetics.mul((Number) l, (Number) r); + } + l = unwrapJodaTime(l); + r = unwrapJodaTime(r); + if (l instanceof Number && r instanceof IntervalYearMonth) { + return ((IntervalYearMonth) r).mul(((Number) l).intValue()); + } + if (r instanceof Number && l instanceof IntervalYearMonth) { + return ((IntervalYearMonth) l).mul(((Number) r).intValue()); + } + if (l instanceof Number && r instanceof IntervalDayTime) { + return ((IntervalDayTime) r).mul(((Number) l).longValue()); + } + if (r instanceof Number && l instanceof IntervalDayTime) { + return ((IntervalDayTime) l).mul(((Number) r).longValue()); + } + + throw new SqlIllegalArgumentException("Cannot compute [*] between [{}] [{}]", l.getClass().getSimpleName(), + r.getClass().getSimpleName()); + }, "*"), DIV(Arithmetics::div, "/"), MOD(Arithmetics::mod, "%"); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/Mul.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/Mul.java index edfea25d5c0..40e4bdfaaed 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/Mul.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/Mul.java @@ -9,16 +9,55 @@ import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.BinaryArithmeticProcessor.BinaryArithmeticOperation; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DataTypes; + +import static org.elasticsearch.common.logging.LoggerMessageFormat.format; /** * Multiplication function ({@code a * b}). */ public class Mul extends ArithmeticOperation { + private DataType dataType; + public Mul(Location location, Expression left, Expression right) { super(location, left, right, BinaryArithmeticOperation.MUL); } + @Override + protected TypeResolution resolveType() { + if (!childrenResolved()) { + return new TypeResolution("Unresolved children"); + } + + DataType l = left().dataType(); + DataType r = right().dataType(); + + // 1. both are numbers + if (l.isNumeric() && r.isNumeric()) { + return TypeResolution.TYPE_RESOLVED; + } + + if (DataTypes.isInterval(l) && r.isInteger()) { + dataType = l; + return TypeResolution.TYPE_RESOLVED; + } else if (DataTypes.isInterval(r) && l.isInteger()) { + dataType = r; + return TypeResolution.TYPE_RESOLVED; + } + + return new TypeResolution(format("[{}] has arguments with incompatible types [{}] and [{}]", symbol(), l, r)); + } + + @Override + public DataType dataType() { + if (dataType == null) { + dataType = super.dataType(); + } + return dataType; + } + @Override protected NodeInfo info() { return NodeInfo.create(this, Mul::new, left(), right()); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java index e8ce98a8e26..ffd12ea6fb9 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.sql.optimizer; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; +import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer.CleanAliases; import org.elasticsearch.xpack.sql.expression.Alias; import org.elasticsearch.xpack.sql.expression.Attribute; import org.elasticsearch.xpack.sql.expression.AttributeMap; @@ -110,11 +111,6 @@ public class Optimizer extends RuleExecutor { @Override protected Iterable.Batch> batches() { - Batch resolution = new Batch("Finish Analysis", - new PruneSubqueryAliases(), - CleanAliases.INSTANCE - ); - Batch aggregate = new Batch("Aggregation", new PruneDuplicatesInGroupBy(), new ReplaceDuplicateAggsWithReferences(), @@ -162,70 +158,10 @@ public class Optimizer extends RuleExecutor { Batch label = new Batch("Set as Optimized", Limiter.ONCE, new SetAsOptimized()); - return Arrays.asList(resolution, aggregate, operators, local, label); + return Arrays.asList(aggregate, operators, local, label); } - static class PruneSubqueryAliases extends OptimizerRule { - - PruneSubqueryAliases() { - super(TransformDirection.UP); - } - - @Override - protected LogicalPlan rule(SubQueryAlias alias) { - return alias.child(); - } - } - - static class CleanAliases extends OptimizerRule { - - private static final CleanAliases INSTANCE = new CleanAliases(); - - CleanAliases() { - super(TransformDirection.UP); - } - - @Override - protected LogicalPlan rule(LogicalPlan plan) { - if (plan instanceof Project) { - Project p = (Project) plan; - return new Project(p.location(), p.child(), cleanExpressions(p.projections())); - } - - if (plan instanceof Aggregate) { - Aggregate a = (Aggregate) plan; - // clean group expressions - List cleanedGroups = a.groupings().stream().map(CleanAliases::trimAliases).collect(toList()); - return new Aggregate(a.location(), a.child(), cleanedGroups, cleanExpressions(a.aggregates())); - } - - return plan.transformExpressionsOnly(e -> { - if (e instanceof Alias) { - return ((Alias) e).child(); - } - return e; - }); - } - - private List cleanExpressions(List args) { - return args.stream().map(CleanAliases::trimNonTopLevelAliases).map(NamedExpression.class::cast) - .collect(toList()); - } - - static Expression trimNonTopLevelAliases(Expression e) { - if (e instanceof Alias) { - Alias a = (Alias) e; - return new Alias(a.location(), a.name(), a.qualifier(), trimAliases(a.child()), a.id()); - } - return trimAliases(e); - } - - private static Expression trimAliases(Expression e) { - return e.transformDown(Alias::child, Alias.class); - } - } - static class PruneDuplicatesInGroupBy extends OptimizerRule { @Override diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/AbstractBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/AbstractBuilder.java index 480d22a9699..67721b35d7e 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/AbstractBuilder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/AbstractBuilder.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.sql.parser; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.TerminalNode; import org.elasticsearch.xpack.sql.plan.logical.LogicalPlan; @@ -85,6 +86,12 @@ abstract class AbstractBuilder extends SqlBaseBaseVisitor { return new Location(token.getLine(), token.getCharPositionInLine()); } + static String text(ParserRuleContext parserRuleContext) { + Check.notNull(parserRuleContext, "parserRuleContext is null"); + Interval interval = new Interval(parserRuleContext.start.getStartIndex(), parserRuleContext.stop.getStopIndex()); + return parserRuleContext.start.getInputStream().getText(interval); + } + /** * Retrieves the raw text of the node (without interpreting it as a string literal). */ diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java index 6db3b9631de..cd1cb189b6a 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java @@ -63,6 +63,7 @@ import org.elasticsearch.xpack.sql.parser.SqlBaseParser.BuiltinDateTimeFunctionC import org.elasticsearch.xpack.sql.parser.SqlBaseParser.CastExpressionContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.CastTemplateContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ComparisonContext; +import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ConstantDefaultContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ConvertTemplateContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.DateEscapedLiteralContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.DecimalLiteralContext; @@ -103,6 +104,7 @@ import org.elasticsearch.xpack.sql.parser.SqlBaseParser.SubqueryExpressionContex import org.elasticsearch.xpack.sql.parser.SqlBaseParser.SysTypesContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.TimeEscapedLiteralContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.TimestampEscapedLiteralContext; +import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ValueExpressionDefaultContext; import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.type.DataType; @@ -546,9 +548,7 @@ abstract class ExpressionBuilder extends IdentifierBuilder { } @Override - public Literal visitIntervalLiteral(IntervalLiteralContext ctx) { - - IntervalContext interval = ctx.interval(); + public Literal visitInterval(IntervalContext interval) { TimeUnit leading = visitIntervalField(interval.leading); TimeUnit trailing = visitIntervalField(interval.trailing); @@ -571,10 +571,31 @@ abstract class ExpressionBuilder extends IdentifierBuilder { DataType intervalType = Intervals.intervalType(source(interval), leading, trailing); - boolean negative = interval.sign != null && interval.sign.getType() == SqlBaseParser.MINUS; + // negation outside the interval - use xor + boolean negative = false; + + ParserRuleContext parentCtx = interval.getParent(); + if (parentCtx != null) { + if (parentCtx instanceof IntervalLiteralContext) { + parentCtx = parentCtx.getParent(); + if (parentCtx instanceof ConstantDefaultContext) { + parentCtx = parentCtx.getParent(); + if (parentCtx instanceof ValueExpressionDefaultContext) { + parentCtx = parentCtx.getParent(); + if (parentCtx instanceof ArithmeticUnaryContext) { + ArithmeticUnaryContext auc = (ArithmeticUnaryContext) parentCtx; + negative = auc.MINUS() != null; + } + } + } + } + } + + + // negation inside the interval + negative ^= interval.sign != null && interval.sign.getType() == SqlBaseParser.MINUS; TemporalAmount value = null; - String valueAsText = null; if (interval.valueNumeric != null) { if (trailing != null) { @@ -583,18 +604,14 @@ abstract class ExpressionBuilder extends IdentifierBuilder { + "use the string notation instead", trailing); } value = of(interval.valueNumeric, leading); - valueAsText = interval.valueNumeric.getText(); } else { value = of(interval.valuePattern, negative, intervalType); - valueAsText = interval.valuePattern.getText(); } - String name = "INTERVAL " + valueAsText + " " + leading.name() + (trailing != null ? " TO " + trailing.name() : ""); - Interval timeInterval = value instanceof Period ? new IntervalYearMonth((Period) value, intervalType) : new IntervalDayTime((Duration) value, intervalType); - return new Literal(source(ctx), name, timeInterval, intervalType); + return new Literal(source(interval), text(interval), timeInterval, timeInterval.dataType()); } private TemporalAmount of(NumberContext valueNumeric, TimeUnit unit) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java index 57c2d4156af..35ba50ab75a 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFuncti import org.elasticsearch.xpack.sql.expression.function.aggregate.CompoundNumericAggregate; import org.elasticsearch.xpack.sql.expression.function.aggregate.Count; import org.elasticsearch.xpack.sql.expression.function.aggregate.InnerAggregate; +import org.elasticsearch.xpack.sql.expression.function.grouping.GroupingFunction; import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunctionAttribute; import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeFunction; @@ -336,6 +337,11 @@ class QueryFolder extends RuleExecutor { TimeZone dt = DataType.DATE == child.dataType() ? UTC : null; queryC = queryC.addColumn(new GroupByRef(matchingGroup.id(), null, dt)); } + // handle histogram + else if (child instanceof GroupingFunction) { + queryC = queryC.addColumn(new GroupByRef(matchingGroup.id(), null, null)); + } + // fallback to regular agg functions else { // the only thing left is agg function Check.isTrue(Functions.isAggregate(child), diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java index 2c1f5a1e449..23352af790d 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.expression.ExpressionId; import org.elasticsearch.xpack.sql.expression.Expressions; import org.elasticsearch.xpack.sql.expression.FieldAttribute; +import org.elasticsearch.xpack.sql.expression.Foldables; import org.elasticsearch.xpack.sql.expression.Literal; import org.elasticsearch.xpack.sql.expression.NamedExpression; import org.elasticsearch.xpack.sql.expression.function.Function; @@ -27,9 +28,12 @@ import org.elasticsearch.xpack.sql.expression.function.aggregate.PercentileRanks import org.elasticsearch.xpack.sql.expression.function.aggregate.Percentiles; import org.elasticsearch.xpack.sql.expression.function.aggregate.Stats; import org.elasticsearch.xpack.sql.expression.function.aggregate.Sum; +import org.elasticsearch.xpack.sql.expression.function.grouping.GroupingFunction; +import org.elasticsearch.xpack.sql.expression.function.grouping.Histogram; import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeFunction; import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeHistogramFunction; +import org.elasticsearch.xpack.sql.expression.literal.Intervals; import org.elasticsearch.xpack.sql.expression.predicate.Range; import org.elasticsearch.xpack.sql.expression.predicate.fulltext.MatchQueryPredicate; import org.elasticsearch.xpack.sql.expression.predicate.fulltext.MultiMatchQueryPredicate; @@ -57,10 +61,10 @@ import org.elasticsearch.xpack.sql.querydsl.agg.AndAggFilter; import org.elasticsearch.xpack.sql.querydsl.agg.AvgAgg; import org.elasticsearch.xpack.sql.querydsl.agg.CardinalityAgg; import org.elasticsearch.xpack.sql.querydsl.agg.ExtendedStatsAgg; -import org.elasticsearch.xpack.sql.querydsl.agg.GroupByColumnKey; -import org.elasticsearch.xpack.sql.querydsl.agg.GroupByDateKey; +import org.elasticsearch.xpack.sql.querydsl.agg.GroupByDateHistogram; import org.elasticsearch.xpack.sql.querydsl.agg.GroupByKey; -import org.elasticsearch.xpack.sql.querydsl.agg.GroupByScriptKey; +import org.elasticsearch.xpack.sql.querydsl.agg.GroupByNumericHistogram; +import org.elasticsearch.xpack.sql.querydsl.agg.GroupByValue; import org.elasticsearch.xpack.sql.querydsl.agg.LeafAgg; import org.elasticsearch.xpack.sql.querydsl.agg.MatrixStatsAgg; import org.elasticsearch.xpack.sql.querydsl.agg.MaxAgg; @@ -85,6 +89,7 @@ import org.elasticsearch.xpack.sql.querydsl.query.TermQuery; import org.elasticsearch.xpack.sql.querydsl.query.TermsQuery; import org.elasticsearch.xpack.sql.querydsl.query.WildcardQuery; import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.util.Check; import org.elasticsearch.xpack.sql.util.ReflectionUtils; @@ -231,10 +236,16 @@ final class QueryTranslator { Map aggMap = new LinkedHashMap<>(); for (Expression exp : groupings) { + GroupByKey key = null; + ExpressionId id; String aggId; + if (exp instanceof NamedExpression) { NamedExpression ne = (NamedExpression) exp; + id = ne.id(); + aggId = id.toString(); + // change analyzed to non non-analyzed attributes if (exp instanceof FieldAttribute) { FieldAttribute fa = (FieldAttribute) exp; @@ -242,21 +253,51 @@ final class QueryTranslator { ne = fa.exactAttribute(); } } - aggId = ne.id().toString(); - - GroupByKey key; // handle functions differently if (exp instanceof Function) { // dates are handled differently because of date histograms if (exp instanceof DateTimeHistogramFunction) { DateTimeHistogramFunction dthf = (DateTimeHistogramFunction) exp; - key = new GroupByDateKey(aggId, nameOf(exp), dthf.interval(), dthf.timeZone()); + key = new GroupByDateHistogram(aggId, nameOf(exp), dthf.interval(), dthf.timeZone()); } // all other scalar functions become a script else if (exp instanceof ScalarFunction) { ScalarFunction sf = (ScalarFunction) exp; - key = new GroupByScriptKey(aggId, nameOf(exp), sf.asScript()); + key = new GroupByValue(aggId, sf.asScript()); + } + // histogram + else if (exp instanceof GroupingFunction) { + if (exp instanceof Histogram) { + Histogram h = (Histogram) exp; + Expression field = h.field(); + + // date histogram + if (h.dataType() == DataType.DATE) { + long intervalAsMillis = Intervals.inMillis(h.interval()); + // TODO: set timezone + if (field instanceof FieldAttribute || field instanceof DateTimeHistogramFunction) { + key = new GroupByDateHistogram(aggId, nameOf(field), intervalAsMillis, h.timeZone()); + } else if (field instanceof Function) { + key = new GroupByDateHistogram(aggId, ((Function) field).asScript(), intervalAsMillis, h.timeZone()); + } + } + // numeric histogram + else { + if (field instanceof FieldAttribute || field instanceof DateTimeHistogramFunction) { + key = new GroupByNumericHistogram(aggId, nameOf(field), Foldables.doubleValueOf(h.interval())); + } else if (field instanceof Function) { + key = new GroupByNumericHistogram(aggId, ((Function) field).asScript(), + Foldables.doubleValueOf(h.interval())); + } + } + if (key == null) { + throw new SqlIllegalArgumentException("Unsupported histogram field {}", field); + } + } + else { + throw new SqlIllegalArgumentException("Unsupproted grouping function {}", exp); + } } // bumped into into an invalid function (which should be caught by the verifier) else { @@ -264,14 +305,14 @@ final class QueryTranslator { } } else { - key = new GroupByColumnKey(aggId, ne.name()); + key = new GroupByValue(aggId, ne.name()); } - - aggMap.put(ne.id(), key); } else { throw new SqlIllegalArgumentException("Don't know how to group on {}", exp.nodeString()); } + + aggMap.put(id, key); } return new GroupingContext(aggMap); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlClearCursorAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlClearCursorAction.java index 00e316ffabe..cf4a66131cf 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlClearCursorAction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlClearCursorAction.java @@ -27,7 +27,7 @@ public class RestSqlClearCursorAction extends BaseRestHandler { private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestSqlClearCursorAction.class)); - RestSqlClearCursorAction(Settings settings, RestController controller) { + public RestSqlClearCursorAction(Settings settings, RestController controller) { super(settings); // TODO: remove deprecated endpoint in 8.0.0 controller.registerWithDeprecatedHandler( diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java index 2158d0a0037..bd03715ee22 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java @@ -38,7 +38,7 @@ public class RestSqlQueryAction extends BaseRestHandler { private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestSqlQueryAction.class)); - RestSqlQueryAction(Settings settings, RestController controller) { + public RestSqlQueryAction(Settings settings, RestController controller) { super(settings); // TODO: remove deprecated endpoint in 8.0.0 controller.registerWithDeprecatedHandler( @@ -56,8 +56,8 @@ public class RestSqlQueryAction extends BaseRestHandler { SqlQueryRequest sqlRequest; try (XContentParser parser = request.contentOrSourceParamParser()) { sqlRequest = SqlQueryRequest.fromXContent(parser); - } - + } + /* * Since we support {@link TextFormat} and * {@link XContent} outputs we can't use {@link RestToXContentListener} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/Agg.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/Agg.java index 09c92a77ac3..10448eae511 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/Agg.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/Agg.java @@ -27,7 +27,7 @@ public abstract class Agg { return id; } - public String fieldName() { + protected String fieldName() { return fieldName; } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/Aggs.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/Aggs.java index b8faedec718..c7ab17670a2 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/Aggs.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/Aggs.java @@ -10,6 +10,7 @@ import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregati import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; +import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate; import org.elasticsearch.xpack.sql.querydsl.container.Sort.Direction; import java.util.ArrayList; @@ -39,15 +40,15 @@ public class Aggs { public static final String ROOT_GROUP_NAME = "groupby"; - public static final GroupByKey IMPLICIT_GROUP_KEY = new GroupByKey(ROOT_GROUP_NAME, EMPTY, null) { + public static final GroupByKey IMPLICIT_GROUP_KEY = new GroupByKey(ROOT_GROUP_NAME, EMPTY, null, null) { @Override - public CompositeValuesSourceBuilder asValueSource() { + public CompositeValuesSourceBuilder createSourceBuilder() { throw new SqlIllegalArgumentException("Default group does not translate to an aggregation"); } @Override - protected GroupByKey copy(String id, String fieldName, Direction direction) { + protected GroupByKey copy(String id, String fieldName, ScriptTemplate script, Direction direction) { return this; } }; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByColumnKey.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByColumnKey.java deleted file mode 100644 index 931eaee6464..00000000000 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByColumnKey.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.sql.querydsl.agg; - -import org.elasticsearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder; -import org.elasticsearch.xpack.sql.querydsl.container.Sort.Direction; - -/** - * GROUP BY key for regular fields. - */ -public class GroupByColumnKey extends GroupByKey { - - public GroupByColumnKey(String id, String fieldName) { - this(id, fieldName, null); - } - - public GroupByColumnKey(String id, String fieldName, Direction direction) { - super(id, fieldName, direction); - } - - @Override - public TermsValuesSourceBuilder asValueSource() { - return new TermsValuesSourceBuilder(id()) - .field(fieldName()) - .order(direction().asOrder()) - .missingBucket(true); - } - - @Override - protected GroupByKey copy(String id, String fieldName, Direction direction) { - return new GroupByColumnKey(id, fieldName, direction); - } -} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByDateHistogram.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByDateHistogram.java new file mode 100644 index 00000000000..714d5238bb7 --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByDateHistogram.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.querydsl.agg; + +import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; +import org.elasticsearch.search.aggregations.bucket.composite.DateHistogramValuesSourceBuilder; +import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate; +import org.elasticsearch.xpack.sql.querydsl.container.Sort.Direction; +import org.joda.time.DateTimeZone; + +import java.util.Objects; +import java.util.TimeZone; + +/** + * GROUP BY key based on histograms on date fields. + */ +public class GroupByDateHistogram extends GroupByKey { + + private final long interval; + private final TimeZone timeZone; + + public GroupByDateHistogram(String id, String fieldName, long interval, TimeZone timeZone) { + this(id, fieldName, null, null, interval, timeZone); + } + + public GroupByDateHistogram(String id, ScriptTemplate script, long interval, TimeZone timeZone) { + this(id, null, script, null, interval, timeZone); + } + + private GroupByDateHistogram(String id, String fieldName, ScriptTemplate script, Direction direction, long interval, + TimeZone timeZone) { + super(id, fieldName, script, direction); + this.interval = interval; + this.timeZone = timeZone; + + } + + @Override + protected CompositeValuesSourceBuilder createSourceBuilder() { + return new DateHistogramValuesSourceBuilder(id()) + .interval(interval) + .timeZone(DateTimeZone.forTimeZone(timeZone)); + } + + @Override + protected GroupByKey copy(String id, String fieldName, ScriptTemplate script, Direction direction) { + return new GroupByDateHistogram(id, fieldName, script, direction, interval, timeZone); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), interval, timeZone); + } + + @Override + public boolean equals(Object obj) { + if (super.equals(obj)) { + GroupByDateHistogram other = (GroupByDateHistogram) obj; + return Objects.equals(interval, other.interval) + && Objects.equals(timeZone, other.timeZone); + } + return false; + } +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByDateKey.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByDateKey.java deleted file mode 100644 index 61c00c706ee..00000000000 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByDateKey.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.sql.querydsl.agg; - -import org.elasticsearch.search.aggregations.bucket.composite.DateHistogramValuesSourceBuilder; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.xpack.sql.querydsl.container.Sort.Direction; -import org.joda.time.DateTimeZone; - -import java.util.Objects; -import java.util.TimeZone; - -/** - * GROUP BY key specific for date fields. - */ -public class GroupByDateKey extends GroupByKey { - - private final String interval; - private final TimeZone timeZone; - - public GroupByDateKey(String id, String fieldName, String interval, TimeZone timeZone) { - this(id, fieldName, null, interval, timeZone); - } - - public GroupByDateKey(String id, String fieldName, Direction direction, String interval, TimeZone timeZone) { - super(id, fieldName, direction); - this.interval = interval; - this.timeZone = timeZone; - } - - public String interval() { - return interval; - } - - public TimeZone timeZone() { - return timeZone; - } - - @Override - public DateHistogramValuesSourceBuilder asValueSource() { - return new DateHistogramValuesSourceBuilder(id()) - .field(fieldName()) - .dateHistogramInterval(new DateHistogramInterval(interval)) - .timeZone(DateTimeZone.forTimeZone(timeZone)) - .missingBucket(true); - } - - @Override - protected GroupByKey copy(String id, String fieldName, Direction direction) { - return new GroupByDateKey(id, fieldName, direction, interval, timeZone); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), interval, timeZone); - } - - @Override - public boolean equals(Object obj) { - if (super.equals(obj)) { - GroupByDateKey other = (GroupByDateKey) obj; - return Objects.equals(interval, other.interval) - && Objects.equals(timeZone, other.timeZone); - } - return false; - } -} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByKey.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByKey.java index fd2bd5799df..7d74c1c3330 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByKey.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByKey.java @@ -6,7 +6,10 @@ package org.elasticsearch.xpack.sql.querydsl.agg; import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; +import org.elasticsearch.search.aggregations.support.ValueType; +import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate; import org.elasticsearch.xpack.sql.querydsl.container.Sort.Direction; +import org.elasticsearch.xpack.sql.type.DataType; import java.util.Objects; @@ -15,33 +18,64 @@ import java.util.Objects; */ public abstract class GroupByKey extends Agg { - private final Direction direction; + protected final Direction direction; + private final ScriptTemplate script; - GroupByKey(String id, String fieldName, Direction direction) { + protected GroupByKey(String id, String fieldName, ScriptTemplate script, Direction direction) { super(id, fieldName); // ASC is the default order of CompositeValueSource this.direction = direction == null ? Direction.ASC : direction; + this.script = script; } - public Direction direction() { - return direction; + public final CompositeValuesSourceBuilder asValueSource() { + CompositeValuesSourceBuilder builder = createSourceBuilder(); + + if (script != null) { + builder.script(script.toPainless()); + if (script.outputType().isInteger()) { + builder.valueType(ValueType.LONG); + } else if (script.outputType().isRational()) { + builder.valueType(ValueType.DOUBLE); + } else if (script.outputType().isString()) { + builder.valueType(ValueType.STRING); + } else if (script.outputType() == DataType.DATE) { + builder.valueType(ValueType.DATE); + } else if (script.outputType() == DataType.BOOLEAN) { + builder.valueType(ValueType.BOOLEAN); + } else if (script.outputType() == DataType.IP) { + builder.valueType(ValueType.IP); + } + } + // field based + else { + builder.field(fieldName()); + } + return builder.order(direction.asOrder()) + .missingBucket(true); } - public abstract CompositeValuesSourceBuilder asValueSource(); + protected abstract CompositeValuesSourceBuilder createSourceBuilder(); - protected abstract GroupByKey copy(String id, String fieldName, Direction direction); + protected abstract GroupByKey copy(String id, String fieldName, ScriptTemplate script, Direction direction); public GroupByKey with(Direction direction) { - return this.direction == direction ? this : copy(id(), fieldName(), direction); + return this.direction == direction ? this : copy(id(), fieldName(), script, direction); + } + + public ScriptTemplate script() { + return script; } @Override public int hashCode() { - return Objects.hash(id(), fieldName(), direction); + return Objects.hash(id(), fieldName(), script, direction); } @Override public boolean equals(Object obj) { - return super.equals(obj) && Objects.equals(direction, ((GroupByKey) obj).direction); + return super.equals(obj) + && Objects.equals(script, ((GroupByKey) obj).script) + && Objects.equals(direction, ((GroupByKey) obj).direction); } } \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByNumericHistogram.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByNumericHistogram.java new file mode 100644 index 00000000000..a03faede75f --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByNumericHistogram.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.querydsl.agg; + +import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; +import org.elasticsearch.search.aggregations.bucket.composite.HistogramValuesSourceBuilder; +import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate; +import org.elasticsearch.xpack.sql.querydsl.container.Sort.Direction; + +import java.util.Objects; + +/** + * GROUP BY key based on histograms on numeric fields. + */ +public class GroupByNumericHistogram extends GroupByKey { + + private final double interval; + + public GroupByNumericHistogram(String id, String fieldName, double interval) { + this(id, fieldName, null, null, interval); + } + + public GroupByNumericHistogram(String id, ScriptTemplate script, double interval) { + this(id, null, script, null, interval); + } + + private GroupByNumericHistogram(String id, String fieldName, ScriptTemplate script, Direction direction, double interval) { + super(id, fieldName, script, direction); + this.interval = interval; + } + + @Override + protected CompositeValuesSourceBuilder createSourceBuilder() { + return new HistogramValuesSourceBuilder(id()) + .interval(interval); + } + + @Override + protected GroupByKey copy(String id, String fieldName, ScriptTemplate script, Direction direction) { + return new GroupByNumericHistogram(id, fieldName, script, direction, interval); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), interval); + } + + @Override + public boolean equals(Object obj) { + if (super.equals(obj)) { + GroupByNumericHistogram other = (GroupByNumericHistogram) obj; + return interval == other.interval; + } + return false; + } +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByScriptKey.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByScriptKey.java deleted file mode 100644 index 9c907be38da..00000000000 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByScriptKey.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.sql.querydsl.agg; - -import org.elasticsearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate; -import org.elasticsearch.xpack.sql.querydsl.container.Sort.Direction; -import org.elasticsearch.xpack.sql.type.DataType; - -import java.util.Objects; - -/** - * GROUP BY key for scripts (typically caused by functions). - */ -public class GroupByScriptKey extends GroupByKey { - - private final ScriptTemplate script; - - public GroupByScriptKey(String id, String fieldName, ScriptTemplate script) { - this(id, fieldName, null, script); - } - - public GroupByScriptKey(String id, String fieldName, Direction direction, ScriptTemplate script) { - super(id, fieldName, direction); - this.script = script; - } - - public ScriptTemplate script() { - return script; - } - - @Override - public TermsValuesSourceBuilder asValueSource() { - TermsValuesSourceBuilder builder = new TermsValuesSourceBuilder(id()) - .script(script.toPainless()) - .order(direction().asOrder()) - .missingBucket(true); - - if (script.outputType().isInteger()) { - builder.valueType(ValueType.LONG); - } else if (script.outputType().isRational()) { - builder.valueType(ValueType.DOUBLE); - } else if (script.outputType().isString()) { - builder.valueType(ValueType.STRING); - } else if (script.outputType() == DataType.DATE) { - builder.valueType(ValueType.DATE); - } else if (script.outputType() == DataType.BOOLEAN) { - builder.valueType(ValueType.BOOLEAN); - } else if (script.outputType() == DataType.IP) { - builder.valueType(ValueType.IP); - } - - return builder; - } - - @Override - protected GroupByKey copy(String id, String fieldName, Direction direction) { - return new GroupByScriptKey(id, fieldName, direction, script); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), script); - } - - @Override - public boolean equals(Object obj) { - return super.equals(obj) && Objects.equals(script, ((GroupByScriptKey) obj).script); - } -} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByValue.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByValue.java new file mode 100644 index 00000000000..12e09dbc52d --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByValue.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.querydsl.agg; + +import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; +import org.elasticsearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder; +import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate; +import org.elasticsearch.xpack.sql.querydsl.container.Sort.Direction; + +/** + * GROUP BY key for fields or scripts. + */ +public class GroupByValue extends GroupByKey { + + public GroupByValue(String id, String fieldName) { + this(id, fieldName, null, null); + } + + public GroupByValue(String id, ScriptTemplate script) { + this(id, null, script, null); + } + + private GroupByValue(String id, String fieldName, ScriptTemplate script, Direction direction) { + super(id, fieldName, script, direction); + } + + @Override + protected CompositeValuesSourceBuilder createSourceBuilder() { + return new TermsValuesSourceBuilder(id()); + } + + @Override + protected GroupByKey copy(String id, String fieldName, ScriptTemplate script, Direction direction) { + return new GroupByValue(id, fieldName, script, direction); + } +} diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java index 46e19bd349c..27ba84192f3 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java @@ -178,11 +178,24 @@ public class VerifierErrorMessagesTests extends ESTestCase { } // GROUP BY + public void testGroupBySelectWithAlias() { + assertNotNull(accept("SELECT int AS i FROM test GROUP BY i")); + } + + public void testGroupBySelectWithAliasOrderOnActualField() { + assertNotNull(accept("SELECT int AS i FROM test GROUP BY i ORDER BY int")); + } + public void testGroupBySelectNonGrouped() { assertEquals("1:8: Cannot use non-grouped column [text], expected [int]", error("SELECT text, int FROM test GROUP BY int")); } + public void testGroupByFunctionSelectFieldFromGroupByFunction() { + assertEquals("1:8: Cannot use non-grouped column [int], expected [ABS(int)]", + error("SELECT int FROM test GROUP BY ABS(int)")); + } + public void testGroupByOrderByNonGrouped() { assertEquals("1:50: Cannot order by non-grouped column [bool], expected [text]", error("SELECT MAX(int) FROM test GROUP BY text ORDER BY bool")); @@ -203,6 +216,11 @@ public class VerifierErrorMessagesTests extends ESTestCase { error("SELECT MAX(int) FROM test GROUP BY text ORDER BY YEAR(date)")); } + public void testGroupByOrderByFieldFromGroupByFunction() { + assertEquals("1:54: Cannot use non-grouped column [int], expected [ABS(int)]", + error("SELECT ABS(int) FROM test GROUP BY ABS(int) ORDER BY int")); + } + public void testGroupByOrderByScalarOverNonGrouped_WithHaving() { assertEquals("1:71: Cannot order by non-grouped column [YEAR(date [UTC])], expected [text]", error("SELECT MAX(int) FROM test GROUP BY text HAVING MAX(int) > 10 ORDER BY YEAR(date)")); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceGeneratorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceGeneratorTests.java index ae7830ad6d9..75f37f8e71f 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceGeneratorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceGeneratorTests.java @@ -16,7 +16,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.sql.expression.FieldAttribute; import org.elasticsearch.xpack.sql.expression.function.Score; import org.elasticsearch.xpack.sql.querydsl.agg.AvgAgg; -import org.elasticsearch.xpack.sql.querydsl.agg.GroupByColumnKey; +import org.elasticsearch.xpack.sql.querydsl.agg.GroupByValue; import org.elasticsearch.xpack.sql.querydsl.container.AttributeSort; import org.elasticsearch.xpack.sql.querydsl.container.QueryContainer; import org.elasticsearch.xpack.sql.querydsl.container.ScoreSort; @@ -62,7 +62,7 @@ public class SourceGeneratorTests extends ESTestCase { } public void testLimit() { - QueryContainer container = new QueryContainer().withLimit(10).addGroups(singletonList(new GroupByColumnKey("1", "field"))); + QueryContainer container = new QueryContainer().withLimit(10).addGroups(singletonList(new GroupByValue("1", "field"))); int size = randomIntBetween(1, 10); SearchSourceBuilder sourceBuilder = SourceGenerator.sourceBuilder(container, null, size); Builder aggBuilder = sourceBuilder.aggregations(); @@ -114,7 +114,7 @@ public class SourceGeneratorTests extends ESTestCase { public void testNoSortIfAgg() { QueryContainer container = new QueryContainer() - .addGroups(singletonList(new GroupByColumnKey("group_id", "group_column"))) + .addGroups(singletonList(new GroupByValue("group_id", "group_column"))) .addAgg("group_id", new AvgAgg("agg_id", "avg_column")); SearchSourceBuilder sourceBuilder = SourceGenerator.sourceBuilder(container, null, randomIntBetween(1, 10)); assertNull(sourceBuilder.sorts()); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/TyperResolutionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/TyperResolutionTests.java new file mode 100644 index 00000000000..27bfcaf3722 --- /dev/null +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/TyperResolutionTests.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.expression; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.sql.expression.Expression.TypeResolution; +import org.elasticsearch.xpack.sql.expression.literal.IntervalYearMonth; +import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Mul; +import org.elasticsearch.xpack.sql.type.DataType; + +import java.time.Period; + +import static org.elasticsearch.xpack.sql.tree.Location.EMPTY; + +public class TyperResolutionTests extends ESTestCase { + + public void testMulNumeric() { + Mul m = new Mul(EMPTY, L(1), L(2)); + assertEquals(TypeResolution.TYPE_RESOLVED, m.typeResolved()); + } + + public void testMulIntervalAndNumber() { + Mul m = new Mul(EMPTY, L(1), randomYearInterval()); + assertEquals(TypeResolution.TYPE_RESOLVED, m.typeResolved()); + } + + public void testMulNumberAndInterval() { + Mul m = new Mul(EMPTY, randomYearInterval(), L(1)); + assertEquals(TypeResolution.TYPE_RESOLVED, m.typeResolved()); + } + + public void testMulTypeResolution() throws Exception { + Mul mul = new Mul(EMPTY, randomYearInterval(), randomYearInterval()); + assertTrue(mul.typeResolved().unresolved()); + } + + private static Literal randomYearInterval() { + return Literal.of(EMPTY, new IntervalYearMonth(Period.ofMonths(randomInt(123)), DataType.INTERVAL_YEAR_TO_MONTH)); + } + + private static Literal L(Object value) { + return Literal.of(EMPTY, value); + } +} diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/BinaryArithmeticTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/BinaryArithmeticTests.java index b4f4278b431..748718d0a3a 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/BinaryArithmeticTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/BinaryArithmeticTests.java @@ -157,6 +157,22 @@ public class BinaryArithmeticTests extends ESTestCase { assertEquals(L(now.minus(t)), L(x)); } + public void testMulIntervalNumber() throws Exception { + Literal l = interval(Duration.ofHours(2), INTERVAL_HOUR); + IntervalDayTime interval = mul(l, -1); + assertEquals(INTERVAL_HOUR, interval.dataType()); + Duration p = interval.interval(); + assertEquals(Duration.ofHours(2).negated(), p); + } + + public void testMulNumberInterval() throws Exception { + Literal r = interval(Period.ofYears(1), INTERVAL_YEAR); + IntervalYearMonth interval = mul(-2, r); + assertEquals(INTERVAL_YEAR, interval.dataType()); + Period p = interval.interval(); + assertEquals(Period.ofYears(2).negated(), p); + } + @SuppressWarnings("unchecked") private static T add(Object l, Object r) { Add add = new Add(EMPTY, L(l), L(r)); @@ -171,6 +187,12 @@ public class BinaryArithmeticTests extends ESTestCase { return (T) sub.fold(); } + @SuppressWarnings("unchecked") + private static T mul(Object l, Object r) { + Mul mul = new Mul(EMPTY, L(l), L(r)); + assertTrue(mul.foldable()); + return (T) mul.fold(); + } private static Literal L(Object value) { return Literal.of(EMPTY, value); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java index 6c986b745ca..514c36ddf72 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.sql.optimizer; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer.PruneSubqueryAliases; import org.elasticsearch.xpack.sql.expression.Alias; import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.expression.Expressions; @@ -71,7 +72,6 @@ import org.elasticsearch.xpack.sql.optimizer.Optimizer.ConstantFolding; import org.elasticsearch.xpack.sql.optimizer.Optimizer.FoldNull; import org.elasticsearch.xpack.sql.optimizer.Optimizer.PropagateEquals; import org.elasticsearch.xpack.sql.optimizer.Optimizer.PruneDuplicateFunctions; -import org.elasticsearch.xpack.sql.optimizer.Optimizer.PruneSubqueryAliases; import org.elasticsearch.xpack.sql.optimizer.Optimizer.ReplaceFoldableAttributes; import org.elasticsearch.xpack.sql.optimizer.Optimizer.SimplifyConditional; import org.elasticsearch.xpack.sql.plan.logical.Filter; diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java index 7461078fa90..146a34920ea 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java @@ -14,7 +14,9 @@ import org.elasticsearch.xpack.sql.analysis.index.EsIndex; import org.elasticsearch.xpack.sql.analysis.index.IndexResolution; import org.elasticsearch.xpack.sql.analysis.index.MappingException; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.FieldAttribute; import org.elasticsearch.xpack.sql.expression.function.FunctionRegistry; +import org.elasticsearch.xpack.sql.expression.function.grouping.Histogram; import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate; import org.elasticsearch.xpack.sql.parser.SqlParser; @@ -24,7 +26,6 @@ import org.elasticsearch.xpack.sql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.sql.plan.logical.Project; import org.elasticsearch.xpack.sql.planner.QueryTranslator.QueryTranslation; import org.elasticsearch.xpack.sql.querydsl.agg.AggFilter; -import org.elasticsearch.xpack.sql.querydsl.agg.GroupByScriptKey; import org.elasticsearch.xpack.sql.querydsl.query.ExistsQuery; import org.elasticsearch.xpack.sql.querydsl.query.NotQuery; import org.elasticsearch.xpack.sql.querydsl.query.Query; @@ -33,12 +34,14 @@ import org.elasticsearch.xpack.sql.querydsl.query.ScriptQuery; import org.elasticsearch.xpack.sql.querydsl.query.TermQuery; import org.elasticsearch.xpack.sql.querydsl.query.TermsQuery; import org.elasticsearch.xpack.sql.stats.Metrics; +import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.type.EsField; import org.elasticsearch.xpack.sql.type.TypesTests; import org.elasticsearch.xpack.sql.util.DateUtils; import org.junit.AfterClass; import org.junit.BeforeClass; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.stream.Stream; @@ -403,7 +406,7 @@ public class QueryTranslatorTests extends ESTestCase { assertFalse(condition.foldable()); QueryTranslator.GroupingContext groupingContext = QueryTranslator.groupBy(((Aggregate) p).groupings()); assertNotNull(groupingContext); - ScriptTemplate scriptTemplate = ((GroupByScriptKey) groupingContext.tail).script(); + ScriptTemplate scriptTemplate = groupingContext.tail.script(); assertEquals("InternalSqlScriptUtils.coalesce([InternalSqlScriptUtils.docValue(doc,params.v0),params.v1])", scriptTemplate.toString()); assertEquals("[{v=int}, {v=10}]", scriptTemplate.params().toString()); @@ -416,9 +419,39 @@ public class QueryTranslatorTests extends ESTestCase { assertFalse(condition.foldable()); QueryTranslator.GroupingContext groupingContext = QueryTranslator.groupBy(((Aggregate) p).groupings()); assertNotNull(groupingContext); - ScriptTemplate scriptTemplate = ((GroupByScriptKey) groupingContext.tail).script(); + ScriptTemplate scriptTemplate = groupingContext.tail.script(); assertEquals("InternalSqlScriptUtils.nullif(InternalSqlScriptUtils.docValue(doc,params.v0),params.v1)", scriptTemplate.toString()); assertEquals("[{v=int}, {v=10}]", scriptTemplate.params().toString()); } + public void testGroupByDateHistogram() { + LogicalPlan p = plan("SELECT MAX(int) FROM test GROUP BY HISTOGRAM(int, 1000)"); + assertTrue(p instanceof Aggregate); + Aggregate a = (Aggregate) p; + List groupings = a.groupings(); + assertEquals(1, groupings.size()); + Expression exp = groupings.get(0); + assertEquals(Histogram.class, exp.getClass()); + Histogram h = (Histogram) exp; + assertEquals(1000, h.interval().fold()); + Expression field = h.field(); + assertEquals(FieldAttribute.class, field.getClass()); + assertEquals(DataType.INTEGER, field.dataType()); + } + + + public void testGroupByHistogram() { + LogicalPlan p = plan("SELECT MAX(int) FROM test GROUP BY HISTOGRAM(date, INTERVAL 2 YEARS)"); + assertTrue(p instanceof Aggregate); + Aggregate a = (Aggregate) p; + List groupings = a.groupings(); + assertEquals(1, groupings.size()); + Expression exp = groupings.get(0); + assertEquals(Histogram.class, exp.getClass()); + Histogram h = (Histogram) exp; + assertEquals("+2-0", h.interval().fold().toString()); + Expression field = h.field(); + assertEquals(FieldAttribute.class, field.getClass()); + assertEquals(DataType.DATE, field.dataType()); + } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java index 5281a893a65..963498bb9b6 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.expression.FieldAttribute; import org.elasticsearch.xpack.sql.expression.Literal; +import org.elasticsearch.xpack.sql.expression.LiteralTests; import org.elasticsearch.xpack.sql.expression.UnresolvedAttributeTests; import org.elasticsearch.xpack.sql.expression.function.Function; import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunction; @@ -21,6 +22,7 @@ import org.elasticsearch.xpack.sql.expression.function.aggregate.InnerAggregate; import org.elasticsearch.xpack.sql.expression.function.aggregate.Percentile; import org.elasticsearch.xpack.sql.expression.function.aggregate.PercentileRanks; import org.elasticsearch.xpack.sql.expression.function.aggregate.Percentiles; +import org.elasticsearch.xpack.sql.expression.function.grouping.Histogram; import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.CurrentDateTime; import org.elasticsearch.xpack.sql.expression.gen.pipeline.AggExtractorInput; import org.elasticsearch.xpack.sql.expression.gen.pipeline.BinaryPipesTests; @@ -455,6 +457,10 @@ public class NodeSubclassTests> extends ESTestCas if (argClass == char.class) { return randomFrom('\\', '|', '/', '`'); } + } else if (toBuildClass == Histogram.class) { + if (argClass == Expression.class) { + return LiteralTests.randomLiteral(); + } } else if (toBuildClass == CurrentDateTime.class) { if (argClass == Expression.class) { return Literal.of(LocationTests.randomLocation(), randomInt(9));