From 87293272d83471074c522c4704c3dce44aabfc72 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Tue, 12 Sep 2017 19:13:25 +0300 Subject: [PATCH] Arithmetic * big refactor of Processor by introducing ProcessorDefinition an immutable tree structure used for resolving multiple inputs across folding (in particular for aggregations) which at runtime gets translated into 'compiled' or small Processors Add expression arithmetic, expression folding and type coercion Folding * for literals, scalars and inside the optimizer Type validation happens per type hierarchy (numeric vs decimal) not type Ceil/Floor/Round functions return long/int instead of double ScalarFunction preserves ProcessorDefinition instead of functionId Original commit: elastic/x-pack-elasticsearch@a703f8b455604f1471d1861c13afe2feb84e216c --- .../xpack/sql/cli/ExplainIT.java | 12 +- .../xpack/sql/jdbc/DebugSqlSpec.java | 7 +- .../xpack/sql/jdbc/SqlSpecIT.java | 4 +- .../SpecBaseIntegrationTestCase.java | 7 +- .../src/test/resources/arithmetic.csv-spec | 13 + .../src/test/resources/arithmetic.sql-spec | 65 ++++ sql/jdbc/src/test/resources/columns.csv-spec | 2 +- sql/jdbc/src/test/resources/debug.csv-spec | 10 +- sql/jdbc/src/test/resources/debug.sql-spec | 4 +- sql/jdbc/src/test/resources/math.sql-spec | 5 +- .../xpack/sql/analysis/analyzer/Analyzer.java | 74 +++- .../xpack/sql/analysis/analyzer/Verifier.java | 5 - .../xpack/sql/execution/search/AggValues.java | 126 +++++++ .../execution/search/AggsRowSetCursor.java | 105 +----- .../search/ProcessingHitExtractor.java | 79 ----- .../sql/execution/search/ScrollCursor.java | 4 +- .../xpack/sql/execution/search/Scroller.java | 156 +++++---- .../search/SearchHitRowSetCursor.java | 1 + .../sql/execution/search/SourceGenerator.java | 66 ++-- .../extractor/ComputingHitExtractor.java | 82 +++++ .../{ => extractor}/ConstantExtractor.java | 6 +- .../{ => extractor}/DocValueExtractor.java | 6 +- .../search/extractor/HitExtractor.java | 26 ++ .../HitExtractors.java} | 36 +- .../{ => extractor}/InnerHitExtractor.java | 6 +- .../{ => extractor}/SourceExtractor.java | 12 +- .../sql/expression/BinaryExpression.java | 20 +- .../xpack/sql/expression/BinaryLogic.java | 28 ++ .../xpack/sql/expression/BinaryOperator.java | 29 +- .../xpack/sql/expression/Expression.java | 3 +- .../xpack/sql/expression/Expressions.java | 12 + .../xpack/sql/expression/Literal.java | 3 +- .../xpack/sql/expression/UnaryExpression.java | 5 - .../sql/expression/function/Function.java | 5 - .../sql/expression/function/Functions.java | 73 +--- .../expression/function/aggregate/Avg.java | 6 + .../function/aggregate/NumericAggregate.java | 5 +- .../function/aggregate/Percentile.java | 5 +- .../function/aggregate/PercentileRank.java | 5 +- .../function/aggregate/PercentileRanks.java | 16 - .../function/aggregate/Percentiles.java | 16 - .../function/scalar/BinaryScalarFunction.java | 98 ++++++ .../sql/expression/function/scalar/Cast.java | 29 +- .../function/scalar/CastProcessor.java | 39 ++- .../function/scalar/ColumnProcessor.java | 32 -- .../function/scalar/ComposeProcessor.java | 77 ----- .../scalar/MathFunctionProcessor.java | 64 ---- .../function/scalar/Processors.java | 47 +++ .../function/scalar/ScalarFunction.java | 70 +--- .../scalar/ScalarFunctionAttribute.java | 19 +- .../function/scalar/UnaryScalarFunction.java | 95 ++++++ .../function/scalar/arithmetic/Add.java | 22 ++ .../scalar/arithmetic/ArithmeticFunction.java | 101 ++++++ .../scalar/arithmetic/Arithmetics.java | 105 ++++++ .../arithmetic/BinaryArithmeticProcessor.java | 106 ++++++ .../BinaryArithmeticProcessorDefinition.java | 53 +++ .../function/scalar/arithmetic/Div.java | 29 ++ .../function/scalar/arithmetic/Mod.java | 22 ++ .../function/scalar/arithmetic/Mul.java | 22 ++ .../function/scalar/arithmetic/Neg.java | 68 ++++ .../function/scalar/arithmetic/Sub.java | 22 ++ .../arithmetic/UnaryArithmeticProcessor.java | 72 ++++ .../scalar/datetime/DateTimeExtractor.java | 35 -- .../scalar/datetime/DateTimeFunction.java | 27 +- .../{ => datetime}/DateTimeProcessor.java | 59 +++- .../function/scalar/datetime/DayOfMonth.java | 5 +- .../function/scalar/datetime/DayOfWeek.java | 5 +- .../function/scalar/datetime/DayOfYear.java | 5 +- .../function/scalar/datetime/HourOfDay.java | 5 +- .../function/scalar/datetime/MinuteOfDay.java | 5 +- .../scalar/datetime/MinuteOfHour.java | 1 + .../function/scalar/datetime/MonthOfYear.java | 5 +- .../scalar/datetime/SecondOfMinute.java | 5 +- .../scalar/datetime/WeekOfWeekYear.java | 5 +- .../function/scalar/datetime/Year.java | 9 +- .../expression/function/scalar/math/ACos.java | 9 +- .../expression/function/scalar/math/ASin.java | 9 +- .../expression/function/scalar/math/ATan.java | 9 +- .../expression/function/scalar/math/Abs.java | 11 +- .../expression/function/scalar/math/Cbrt.java | 9 +- .../expression/function/scalar/math/Ceil.java | 16 +- .../expression/function/scalar/math/Cos.java | 9 +- .../expression/function/scalar/math/Cosh.java | 9 +- .../function/scalar/math/Degrees.java | 9 +- .../expression/function/scalar/math/E.java | 5 +- .../expression/function/scalar/math/Exp.java | 9 +- .../function/scalar/math/Expm1.java | 9 +- .../function/scalar/math/Floor.java | 16 +- .../expression/function/scalar/math/Log.java | 9 +- .../function/scalar/math/Log10.java | 9 +- .../function/scalar/math/MathFunction.java | 24 +- .../function/scalar/math/MathProcessor.java | 138 +++++--- .../expression/function/scalar/math/Pi.java | 5 +- .../function/scalar/math/Radians.java | 9 +- .../function/scalar/math/Round.java | 19 +- .../expression/function/scalar/math/Sin.java | 9 +- .../expression/function/scalar/math/Sinh.java | 9 +- .../expression/function/scalar/math/Sqrt.java | 9 +- .../expression/function/scalar/math/Tan.java | 9 +- .../processor/definition/AggPathInput.java | 48 +++ .../processor/definition/AggValueInput.java | 55 +++ .../processor/definition/AttributeInput.java | 17 + .../definition/BinaryProcessorDefinition.java | 29 ++ .../processor/definition/ConstantInput.java | 22 ++ .../definition/HitExtractorInput.java | 23 ++ .../processor/definition/LeafInput.java | 46 +++ .../definition/ProcessorDefinition.java | 28 ++ .../definition/ProcessorDefinitions.java | 32 ++ .../processor/definition/ReferenceInput.java | 16 + .../definition/UnaryProcessorDefinition.java | 60 ++++ .../processor/definition/UnresolvedInput.java | 22 ++ .../processor/runtime/BinaryProcessor.java | 50 +++ .../processor/runtime/ChainingProcessor.java | 70 ++++ .../processor/runtime/ConstantProcessor.java | 65 ++++ .../runtime/HitExtractorProcessor.java | 76 +++++ .../runtime}/MatrixFieldProcessor.java | 32 +- .../scalar/processor/runtime/Processor.java} | 11 +- .../processor/runtime/SuppliedProcessor.java | 37 ++ .../processor/runtime/UnaryProcessor.java | 68 ++++ .../scalar/script/ScriptTemplate.java | 12 +- .../xpack/sql/expression/predicate/And.java | 6 +- .../predicate/BinaryComparison.java | 16 +- .../sql/expression/predicate/GreaterThan.java | 2 +- .../predicate/GreaterThanOrEqual.java | 2 +- .../sql/expression/predicate/LessThan.java | 2 +- .../expression/predicate/LessThanOrEqual.java | 2 +- .../xpack/sql/expression/predicate/Not.java | 2 +- .../xpack/sql/expression/predicate/Or.java | 6 +- .../xpack/sql/expression/regex/Like.java | 8 +- .../xpack/sql/expression/regex/RLike.java | 9 +- .../xpack/sql/optimizer/Optimizer.java | 271 ++++++++++++--- .../xpack/sql/parser/ExpressionBuilder.java | 51 ++- .../xpack/sql/parser/LogicalPlanBuilder.java | 5 +- .../{CatalogTable.java => EsRelation.java} | 6 +- .../sql/plan/logical/FromlessSelect.java | 50 --- .../{Queryless.java => LocalRelation.java} | 12 +- .../xpack/sql/plan/logical/LogicalPlan.java | 6 +- .../{QuerylessExec.java => LocalExec.java} | 17 +- .../xpack/sql/planner/Mapper.java | 14 +- .../xpack/sql/planner/QueryFolder.java | 316 ++++++++++-------- .../xpack/sql/planner/QueryTranslator.java | 6 +- .../xpack/sql/querydsl/container/AggRef.java | 4 +- .../querydsl/container/ColumnReference.java | 21 ++ .../sql/querydsl/container/ComputedRef.java | 46 +++ .../querydsl/container/FieldReference.java | 12 +- .../sql/querydsl/container/ProcessingRef.java | 32 -- .../querydsl/container/QueryContainer.java | 212 +++++++----- .../sql/querydsl/container/TotalCountRef.java | 4 +- .../xpack/sql/session/Cursor.java | 4 +- .../sql/session/SingletonExecutable.java | 45 +++ .../xpack/sql/session/SqlSettings.java | 5 +- .../xpack/sql/tree/NodeUtils.java | 5 +- .../xpack/sql/type/DataTypeConversion.java | 71 +++- .../xpack/sql/type/DataTypes.java | 4 + .../xpack/sql/util/CollectionUtils.java | 7 +- .../xpack/sql/util/StringUtils.java | 18 +- .../search/ProcessingHitExtractorTests.java | 90 ----- .../execution/search/ScrollCursorTests.java | 9 +- .../ConstantExtractorTests.java | 4 +- .../DocValueExtractorTests.java | 4 +- .../InnerHitExtractorTests.java | 4 +- .../ProcessingHitExtractorTests.java | 84 +++++ .../{ => extractor}/SourceExtractorTests.java | 4 +- .../function/scalar/CastProcessorTests.java | 18 +- .../scalar/ComposeProcessorTests.java | 47 --- .../scalar/MathFunctionProcessorTests.java | 45 --- .../BinaryArithmeticProcessorTests.java | 89 +++++ .../DateTimeProcessorTests.java | 21 +- .../scalar/datetime/DayOfYearTests.java | 2 +- .../math/MathFunctionProcessorTests.java | 45 +++ .../runtime/ChainingProcessorTests.java | 49 +++ .../runtime/ConstantProcessorTests.java | 38 +++ .../runtime}/MatrixFieldProcessorTests.java | 10 +- .../sql/type/DataTypeConversionTests.java | 2 +- 174 files changed, 3948 insertions(+), 1626 deletions(-) create mode 100644 sql/jdbc/src/test/resources/arithmetic.csv-spec create mode 100644 sql/jdbc/src/test/resources/arithmetic.sql-spec create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/AggValues.java delete mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/ProcessingHitExtractor.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingHitExtractor.java rename sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/{ => extractor}/ConstantExtractor.java (90%) rename sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/{ => extractor}/DocValueExtractor.java (91%) create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/HitExtractor.java rename sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/{HitExtractor.java => extractor/HitExtractors.java} (52%) rename sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/{ => extractor}/InnerHitExtractor.java (94%) rename sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/{ => extractor}/SourceExtractor.java (92%) create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/BinaryLogic.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/BinaryScalarFunction.java delete mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ColumnProcessor.java delete mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ComposeProcessor.java delete mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/MathFunctionProcessor.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Processors.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/UnaryScalarFunction.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Add.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/ArithmeticFunction.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Arithmetics.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/BinaryArithmeticProcessor.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/BinaryArithmeticProcessorDefinition.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Div.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Mod.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Mul.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Neg.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Sub.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/UnaryArithmeticProcessor.java delete mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeExtractor.java rename sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/{ => datetime}/DateTimeProcessor.java (55%) create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/AggPathInput.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/AggValueInput.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/AttributeInput.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/BinaryProcessorDefinition.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ConstantInput.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/HitExtractorInput.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/LeafInput.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ProcessorDefinition.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ProcessorDefinitions.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ReferenceInput.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/UnaryProcessorDefinition.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/UnresolvedInput.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/BinaryProcessor.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/ChainingProcessor.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/ConstantProcessor.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/HitExtractorProcessor.java rename sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/{ => processor/runtime}/MatrixFieldProcessor.java (71%) rename sql/server/src/main/java/org/elasticsearch/xpack/sql/{querydsl/container/Reference.java => expression/function/scalar/processor/runtime/Processor.java} (51%) create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/SuppliedProcessor.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/UnaryProcessor.java rename sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/{CatalogTable.java => EsRelation.java} (95%) delete mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/FromlessSelect.java rename sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/{Queryless.java => LocalRelation.java} (89%) rename sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/physical/{QuerylessExec.java => LocalExec.java} (82%) create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/ColumnReference.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/ComputedRef.java delete mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/ProcessingRef.java create mode 100644 sql/server/src/main/java/org/elasticsearch/xpack/sql/session/SingletonExecutable.java delete mode 100644 sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/ProcessingHitExtractorTests.java rename sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/{ => extractor}/ConstantExtractorTests.java (93%) rename sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/{ => extractor}/DocValueExtractorTests.java (94%) rename sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/{ => extractor}/InnerHitExtractorTests.java (91%) create mode 100644 sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ProcessingHitExtractorTests.java rename sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/{ => extractor}/SourceExtractorTests.java (95%) delete mode 100644 sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/ComposeProcessorTests.java delete mode 100644 sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/MathFunctionProcessorTests.java create mode 100644 sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/BinaryArithmeticProcessorTests.java rename sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/{ => datetime}/DateTimeProcessorTests.java (67%) create mode 100644 sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathFunctionProcessorTests.java create mode 100644 sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/ChainingProcessorTests.java create mode 100644 sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/ConstantProcessorTests.java rename sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/{ => processor/runtime}/MatrixFieldProcessorTests.java (80%) diff --git a/sql/cli/src/test/java/org/elasticsearch/xpack/sql/cli/ExplainIT.java b/sql/cli/src/test/java/org/elasticsearch/xpack/sql/cli/ExplainIT.java index a1ba094d200..fb881ea7d77 100644 --- a/sql/cli/src/test/java/org/elasticsearch/xpack/sql/cli/ExplainIT.java +++ b/sql/cli/src/test/java/org/elasticsearch/xpack/sql/cli/ExplainIT.java @@ -27,14 +27,14 @@ public class ExplainIT extends CliIntegrationTestCase { assertThat(in.readLine(), startsWith("----------")); assertThat(in.readLine(), startsWith("Project[[test_field{r}#")); assertThat(in.readLine(), startsWith("\\_SubQueryAlias[test]")); - assertThat(in.readLine(), startsWith(" \\_CatalogTable[test][test_field{r}#")); + assertThat(in.readLine(), startsWith(" \\EsRelation[test][test_field{r}#")); assertEquals("", in.readLine()); command("EXPLAIN (PLAN OPTIMIZED) SELECT * FROM test"); assertThat(in.readLine(), containsString("plan")); assertThat(in.readLine(), startsWith("----------")); assertThat(in.readLine(), startsWith("Project[[test_field{r}#")); - assertThat(in.readLine(), startsWith("\\_CatalogTable[test][test_field{r}#")); + assertThat(in.readLine(), startsWith("\\EsRelation[test][test_field{r}#")); assertEquals("", in.readLine()); // TODO in this case we should probably remove the source filtering entirely. Right? It costs but we don't need it. @@ -71,7 +71,7 @@ public class ExplainIT extends CliIntegrationTestCase { assertThat(in.readLine(), startsWith("Project[[i{r}#")); assertThat(in.readLine(), startsWith("\\_Filter[i{r}#")); assertThat(in.readLine(), startsWith(" \\_SubQueryAlias[test]")); - assertThat(in.readLine(), startsWith(" \\_CatalogTable[test][i{r}#")); + assertThat(in.readLine(), startsWith(" \\EsRelation[test][i{r}#")); assertEquals("", in.readLine()); command("EXPLAIN (PLAN OPTIMIZED) SELECT * FROM test WHERE i = 2"); @@ -79,7 +79,7 @@ public class ExplainIT extends CliIntegrationTestCase { assertThat(in.readLine(), startsWith("----------")); assertThat(in.readLine(), startsWith("Project[[i{r}#")); assertThat(in.readLine(), startsWith("\\_Filter[i{r}#")); - assertThat(in.readLine(), startsWith(" \\_CatalogTable[test][i{r}#")); + assertThat(in.readLine(), startsWith(" \\EsRelation[test][i{r}#")); assertEquals("", in.readLine()); command("EXPLAIN (PLAN EXECUTABLE) SELECT * FROM test WHERE i = 2"); @@ -124,14 +124,14 @@ public class ExplainIT extends CliIntegrationTestCase { assertThat(in.readLine(), startsWith("----------")); assertThat(in.readLine(), startsWith("Aggregate[[],[COUNT(1)#")); assertThat(in.readLine(), startsWith("\\_SubQueryAlias[test]")); - assertThat(in.readLine(), startsWith(" \\_CatalogTable[test][i{r}#")); + assertThat(in.readLine(), startsWith(" \\EsRelation[test][i{r}#")); assertEquals("", in.readLine()); command("EXPLAIN (PLAN OPTIMIZED) SELECT COUNT(*) FROM test"); assertThat(in.readLine(), containsString("plan")); assertThat(in.readLine(), startsWith("----------")); assertThat(in.readLine(), startsWith("Aggregate[[],[COUNT(1)#")); - assertThat(in.readLine(), startsWith("\\_CatalogTable[test][i{r}#")); + assertThat(in.readLine(), startsWith("\\EsRelation[test][i{r}#")); assertEquals("", in.readLine()); command("EXPLAIN (PLAN EXECUTABLE) SELECT COUNT(*) FROM test"); diff --git a/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/DebugSqlSpec.java b/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/DebugSqlSpec.java index 0097edcc0c0..3d958e9ce69 100644 --- a/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/DebugSqlSpec.java +++ b/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/DebugSqlSpec.java @@ -9,15 +9,18 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.xpack.sql.jdbc.framework.JdbcTestUtils; +import org.elasticsearch.xpack.sql.jdbc.framework.LocalH2; import java.nio.file.Path; import java.util.List; @TestLogging(JdbcTestUtils.SQL_TRACE) -public abstract class DebugSqlSpec extends SqlSpecIT { +public class DebugSqlSpec extends SqlSpecIT { + public static LocalH2 H2 = new LocalH2(); - @ParametersFactory(shuffle = false, argumentFormatting = SqlSpecIT.PARAM_FORMATTING) + @ParametersFactory(argumentFormatting = PARAM_FORMATTING) public static List readScriptSpec() throws Exception { + Parser parser = specParser(); return readScriptSpec("/debug.sql-spec", parser); } diff --git a/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/SqlSpecIT.java b/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/SqlSpecIT.java index aa2541bcb3c..93f8e30ea05 100644 --- a/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/SqlSpecIT.java +++ b/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/SqlSpecIT.java @@ -37,7 +37,9 @@ public class SqlSpecIT extends SpecBaseIntegrationTestCase { readScriptSpec("/filter.sql-spec", parser), readScriptSpec("/datetime.sql-spec", parser), readScriptSpec("/math.sql-spec", parser), - readScriptSpec("/agg.sql-spec", parser)); + readScriptSpec("/agg.sql-spec", parser), + readScriptSpec("/arithmetic.sql-spec", parser) + ); } // NOCOMMIT: add tests for nested docs when interplug communication is enabled diff --git a/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/framework/SpecBaseIntegrationTestCase.java b/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/framework/SpecBaseIntegrationTestCase.java index 8436d0bc6fc..cdebb71a767 100644 --- a/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/framework/SpecBaseIntegrationTestCase.java +++ b/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/framework/SpecBaseIntegrationTestCase.java @@ -13,6 +13,7 @@ import org.junit.AfterClass; import org.junit.Before; import java.io.IOException; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.sql.Connection; @@ -123,7 +124,11 @@ public abstract class SpecBaseIntegrationTestCase extends JdbcIntegrationTestCas // returns groupName, testName, its line location, its source and the custom object (based on each test parser) protected static List readScriptSpec(String url, Parser parser) throws Exception { - Path source = PathUtils.get(SpecBaseIntegrationTestCase.class.getResource(url).toURI()); + URL resource = SpecBaseIntegrationTestCase.class.getResource(url); + if (resource == null) { + throw new IllegalArgumentException("Cannot find resource " + url); + } + Path source = PathUtils.get(resource.toURI()); String fileName = source.getFileName().toString(); int dot = fileName.indexOf("."); String groupName = dot > 0 ? fileName.substring(0, dot) : fileName; diff --git a/sql/jdbc/src/test/resources/arithmetic.csv-spec b/sql/jdbc/src/test/resources/arithmetic.csv-spec new file mode 100644 index 00000000000..4d8a9fc3fc2 --- /dev/null +++ b/sql/jdbc/src/test/resources/arithmetic.csv-spec @@ -0,0 +1,13 @@ +// +// Arithmetic tests outside H2 +// + +// the standard behavior here is to return the constant for each element +// the weird thing is that an actual query needs to be ran +arithmeticWithFrom +SELECT 5 - 2 x FROM test_emp; + +x +3 +; + diff --git a/sql/jdbc/src/test/resources/arithmetic.sql-spec b/sql/jdbc/src/test/resources/arithmetic.sql-spec new file mode 100644 index 00000000000..5f77ee008e8 --- /dev/null +++ b/sql/jdbc/src/test/resources/arithmetic.sql-spec @@ -0,0 +1,65 @@ +// +// Arithmetic tests +// + +unaryMinus +SELECT - 1 AS x; +plus +SELECT 1 + 1 AS x; +minus +SELECT 1 - 1 AS x; +divide +SELECT 6 / 3 AS x; +multiply +SELECT 2 * 3 AS x; +mod +SELECT 5 % 2 AS x; +operatorsPriority +SELECT 1 + 3 * 4 / 2 - 2 AS x; +operatorsPriorityWithParanthesis +SELECT ((1 + 3) * 2 / (3 - 1)) * 2 AS x; +literalAliasing +SELECT 2 + 3 AS x, 'foo' y; + +// variable scalar arithmetic +scalarVariablePlus +SELECT emp_no + 10000 AS x FROM test_emp; +scalarVariableMinus +SELECT emp_no - 10000 AS x FROM test_emp; +scalarVariableMul +SELECT emp_no * 10000 AS x FROM test_emp; +scalarVariableDiv +SELECT emp_no / 10000 AS x FROM test_emp; +scalarVariableMod +SELECT emp_no % 10000 AS x FROM test_emp; +scalarVariableMultipleInputs +SELECT (emp_no % 10000) + YEAR(hire_date) AS x FROM test_emp; +scalarVariableTwoInputs +SELECT (emp_no % 10000) + YEAR(hire_date) AS x FROM test_emp; +scalarVariableThreeInputs +SELECT ((emp_no % 10000) + YEAR(hire_date)) / MONTH(birth_date) AS x FROM test_emp; +scalarVariableArithmeticAndEntry +SELECT emp_no, emp_no % 10000 AS x FROM test_emp; +scalarVariableTwoInputsAndEntry +SELECT emp_no, (emp_no % 10000) + YEAR(hire_date) AS x FROM test_emp; +scalarVariableThreeInputsAndEntry +SELECT emp_no, ((emp_no % 10000) + YEAR(hire_date)) / MONTH(birth_date) AS x FROM test_emp; + + +// variable scalar agg +aggVariablePlus +SELECT COUNT(*) + 10000 AS x FROM test_emp GROUP BY gender; +aggVariableMinus +SELECT COUNT(*) - 10000 AS x FROM test_emp GROUP BY gender; +aggVariableMul +SELECT COUNT(*) * 2 AS x FROM test_emp GROUP BY gender; +aggVariableDiv +SELECT COUNT(*) / 5000 AS x FROM test_emp GROUP BY gender; +aggVariableMod +SELECT COUNT(*) % 10000 AS x FROM test_emp GROUP BY gender; +aggVariableTwoInputs +SELECT MAX(emp_no) - MIN(emp_no) AS x FROM test_emp GROUP BY gender; +aggVariableThreeInputs +SELECT (MAX(emp_no) - MIN(emp_no)) + AVG(emp_no) AS x FROM test_emp GROUP BY gender; + + diff --git a/sql/jdbc/src/test/resources/columns.csv-spec b/sql/jdbc/src/test/resources/columns.csv-spec index fdb29abb61b..331c4a705ff 100644 --- a/sql/jdbc/src/test/resources/columns.csv-spec +++ b/sql/jdbc/src/test/resources/columns.csv-spec @@ -9,7 +9,7 @@ columnDetectionOverride SELECT gender, FLOOR(PERCENTILE(emp_no, 97.76)) p1 FROM test_emp GROUP BY gender; -gender:s | p1:double +gender:s | p1:l M | 10095 F | 10099 ; \ No newline at end of file diff --git a/sql/jdbc/src/test/resources/debug.csv-spec b/sql/jdbc/src/test/resources/debug.csv-spec index 19aaf3b2827..d999f38abe9 100644 --- a/sql/jdbc/src/test/resources/debug.csv-spec +++ b/sql/jdbc/src/test/resources/debug.csv-spec @@ -3,9 +3,11 @@ // debug -SELECT gender, PERCENTILE(emp_no, 97.76) p1, PERCENTILE(emp_no, 93.3) p2, PERCENTILE_RANK(emp_no, 10025) rank FROM test_emp GROUP BY gender; +// resolution should happen on the adjiacent nodes as well +//SELECT 1+2+3 x, x + 3 AS y, y FROM test_emp; -gender | p1 | p2 | rank -M | 10095.6112 | 10090.846 | 23.41269841269841 -F | 10099.1936 | 10096.351999999999 | 26.351351351351347 +SELECT 2 + 3 x, 'foo', x + 1 z; + +x | 'foo' | z +5 | 'foo' | 6 ; \ No newline at end of file diff --git a/sql/jdbc/src/test/resources/debug.sql-spec b/sql/jdbc/src/test/resources/debug.sql-spec index 246503514c4..961c6e329bf 100644 --- a/sql/jdbc/src/test/resources/debug.sql-spec +++ b/sql/jdbc/src/test/resources/debug.sql-spec @@ -3,4 +3,6 @@ // debug -SELECT * FROM test_emp WHERE emp_no IS NULL ORDER BY emp_no LIMIT 5 ; +SELECT emp_no, CAST(CEIL(emp_no) AS INT) m, first_name FROM "test_emp" WHERE CEIL(emp_no) < 10010 ORDER BY CEIL(emp_no); +//SELECT YEAR(birth_date) AS d, CAST(SUM(emp_no) AS INT) s FROM "test_emp" GROUP BY YEAR(birth_date) ORDER BY YEAR(birth_date) LIMIT 5; +//SELECT emp_no, SIN(emp_no) + emp_no % 10000 + YEAR(hire_date) / 1000 AS s, emp_no AS y FROM test_emp WHERE emp_no = 10010; diff --git a/sql/jdbc/src/test/resources/math.sql-spec b/sql/jdbc/src/test/resources/math.sql-spec index 4304ad404c3..bcc73c852a0 100644 --- a/sql/jdbc/src/test/resources/math.sql-spec +++ b/sql/jdbc/src/test/resources/math.sql-spec @@ -13,7 +13,8 @@ SELECT ATAN(emp_no) m, first_name FROM "test_emp" WHERE emp_no < 10010 ORDER BY //mathCbrt //SELECT CBRT(emp_no) m, first_name FROM "test_emp" WHERE emp_no < 10010 ORDER BY emp_no; mathCeil -SELECT CEIL(emp_no) m, first_name FROM "test_emp" WHERE emp_no < 10010 ORDER BY emp_no; +// H2 returns CEIL as a double despite the value being an integer; we return a long as the other DBs +SELECT CAST(CEIL(emp_no) AS INT) m, first_name FROM "test_emp" WHERE emp_no < 10010 ORDER BY emp_no; mathCos SELECT COS(emp_no) m, first_name FROM "test_emp" WHERE emp_no < 10010 ORDER BY emp_no; mathCosh @@ -62,7 +63,7 @@ SELECT emp_no, ASIN(emp_no) m, first_name FROM "test_emp" WHERE ASIN(emp_no) < 1 //mathATanFilterAndOrder //SELECT emp_no, ATAN(emp_no) m, first_name FROM "test_emp" WHERE ATAN(emp_no) < 10010 ORDER BY ATAN(emp_no); mathCeilFilterAndOrder -SELECT emp_no, CEIL(emp_no) m, first_name FROM "test_emp" WHERE CEIL(emp_no) < 10010 ORDER BY CEIL(emp_no); +SELECT emp_no, CAST(CEIL(emp_no) AS INT) m, first_name FROM "test_emp" WHERE CEIL(emp_no) < 10010 ORDER BY CEIL(emp_no); //mathCosFilterAndOrder //SELECT emp_no, COS(emp_no) m, first_name FROM "test_emp" WHERE COS(emp_no) < 10010 ORDER BY COS(emp_no); //mathCoshFilterAndOrder diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java index d37b526cc46..acf2d376b9f 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java @@ -34,11 +34,13 @@ import org.elasticsearch.xpack.sql.expression.function.Functions; import org.elasticsearch.xpack.sql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.sql.expression.function.aggregate.Count; import org.elasticsearch.xpack.sql.expression.function.scalar.Cast; +import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.ArithmeticFunction; import org.elasticsearch.xpack.sql.plan.TableIdentifier; import org.elasticsearch.xpack.sql.plan.logical.Aggregate; -import org.elasticsearch.xpack.sql.plan.logical.CatalogTable; +import org.elasticsearch.xpack.sql.plan.logical.EsRelation; import org.elasticsearch.xpack.sql.plan.logical.Filter; import org.elasticsearch.xpack.sql.plan.logical.Join; +import org.elasticsearch.xpack.sql.plan.logical.LocalRelation; import org.elasticsearch.xpack.sql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.sql.plan.logical.OrderBy; import org.elasticsearch.xpack.sql.plan.logical.Project; @@ -49,7 +51,10 @@ import org.elasticsearch.xpack.sql.rule.Rule; import org.elasticsearch.xpack.sql.rule.RuleExecutor; import org.elasticsearch.xpack.sql.session.SqlSession; import org.elasticsearch.xpack.sql.tree.Node; +import org.elasticsearch.xpack.sql.tree.NodeUtils; import org.elasticsearch.xpack.sql.type.CompoundDataType; +import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DataTypeConversion; import org.elasticsearch.xpack.sql.util.StringUtils; import java.util.ArrayList; @@ -90,7 +95,8 @@ public class Analyzer extends RuleExecutor { new ResolveFunctions(), new ResolveAliases(), new ProjectedAggregations(), - new ResolveAggsInHavingAndOrderBy() + new ResolveAggsInHavingAndOrderBy() + //new ImplicitCasting() ); // TODO: this might be removed since the deduplication happens already in ResolveFunctions Batch deduplication = new Batch("Deduplication", @@ -226,6 +232,11 @@ public class Analyzer extends RuleExecutor { } return ur; } + // inlined queries (SELECT 1 + 2) are already resolved + else if (p instanceof LocalRelation) { + return p; + } + return p.transformExpressionsDown(e -> { if (e instanceof SubQueryExpression) { SubQueryExpression sq = (SubQueryExpression) e; @@ -234,6 +245,11 @@ public class Analyzer extends RuleExecutor { return e; }); } + + @Override + protected boolean skipResolved() { + return false; + } } private class ResolveTable extends AnalyzeRule { @@ -250,7 +266,7 @@ public class Analyzer extends RuleExecutor { throw new UnknownIndexException(table.index(), plan); } - LogicalPlan catalogTable = new CatalogTable(plan.location(), found); + LogicalPlan catalogTable = new EsRelation(plan.location(), found); SubQueryAlias sa = new SubQueryAlias(plan.location(), catalogTable, table.index()); if (plan.alias() != null) { @@ -466,7 +482,7 @@ public class Analyzer extends RuleExecutor { if (ordinal != null) { changed = true; if (ordinal > 0 && ordinal <= max) { - NamedExpression reference = aggregates.get(ordinal); + NamedExpression reference = aggregates.get(ordinal - 1); if (containsAggregate(reference)) { throw new AnalysisException(exp, "Group ordinal %d refers to an aggregate function %s which is not compatible/allowed with GROUP BY", ordinal, reference.nodeName()); } @@ -724,8 +740,8 @@ public class Analyzer extends RuleExecutor { } if (child instanceof Cast) { Cast c = (Cast) child; - if (c.argument() instanceof NamedExpression) { - return new Alias(c.location(), ((NamedExpression) c.argument()).name(), c); + if (c.field() instanceof NamedExpression) { + return new Alias(c.location(), ((NamedExpression) c.field()).name(), c); } } //TODO: maybe add something closer to SQL @@ -966,6 +982,52 @@ public class Analyzer extends RuleExecutor { } } + private class ImplicitCasting extends AnalyzeRule { + + @Override + protected boolean skipResolved() { + return false; + } + + @Override + protected LogicalPlan rule(LogicalPlan plan) { + return plan.transformExpressionsDown(this::implicitCast); + } + + private Expression implicitCast(Expression e) { + if (!e.childrenResolved()) { + return e; + } + + Expression left = null, right = null; + + // BinaryOperations are ignored as they are pushed down to ES + // and casting (and thus Aliasing when folding) gets in the way + + if (e instanceof ArithmeticFunction) { + ArithmeticFunction f = (ArithmeticFunction) e; + left = f.left(); + right = f.right(); + } + + if (left != null) { + DataType l = left.dataType(); + DataType r = right.dataType(); + if (!l.same(r)) { + DataType common = DataTypeConversion.commonType(l, r); + if (common == null) { + return e; + } + left = l.same(common) ? left : new Cast(left.location(), left, common); + right = r.same(common) ? right : new Cast(right.location(), right, common); + return NodeUtils.copyTree(e, Arrays.asList(left, right)); + } + } + + return e; + } + } + abstract static class AnalyzeRule extends Rule { // transformUp (post-order) - that is first children and then the node diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index 3e32674ba17..a1159aafca9 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -11,11 +11,9 @@ import org.elasticsearch.xpack.sql.expression.Expressions; import org.elasticsearch.xpack.sql.expression.NamedExpression; import org.elasticsearch.xpack.sql.expression.function.Functions; import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunction; -import org.elasticsearch.xpack.sql.expression.function.scalar.Cast; import org.elasticsearch.xpack.sql.plan.logical.Aggregate; import org.elasticsearch.xpack.sql.plan.logical.Filter; import org.elasticsearch.xpack.sql.plan.logical.LogicalPlan; -import org.elasticsearch.xpack.sql.plan.logical.Project; import org.elasticsearch.xpack.sql.tree.Node; import java.util.ArrayList; @@ -103,9 +101,6 @@ abstract class Verifier { else if (ae instanceof Attribute && !ae.resolved()) { localFailures.add(fail(e, "Cannot resolved '%s' from columns %s", Expressions.name(ae), p.intputSet())); } - else if (ae instanceof Cast && !(p instanceof Project || p instanceof Aggregate)) { - localFailures.add(fail(ae, "Cast is (currently) only supported in SELECT and GROUP BY; not in %s", p.nodeName())); - } })); // consider only nodes that are by themselves unresolved (to avoid unresolved dependencies) diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/AggValues.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/AggValues.java new file mode 100644 index 00000000000..39f29930378 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/AggValues.java @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.execution.search; + +import java.util.Arrays; +import java.util.List; + +/** + * Aggregations are returned by Elasticsearch in a tree structure where each nested level can have a different size. + * For example a group by a, b, c results in 3-level nested array where each level contains all the relevant values + * for its parent entry. + * Assuming there's a total of 2 A's, 3 B's and 5 C's, the values will be + * A-agg level = { A1, A2 } + * B-agg level = { { A1B1, A1B2, A1B3 }, { A2B1, A2B2, A2B3 } + * C-agg level = { { { A1B1C1, A1B1C2 ..}, { A1B2C1, etc... } } } and so on + * + * Further more the columns are added in the order in which they are requested (0, 1, 2) eliminating the need for keys as these are implicit (their position in the list). + * + * To help with the iteration, there are two dedicated counters : + * one that carries (increments) the counter for each level (indicated by the position inside the array) once the children reach their max + * a flat cursor to indicate the row + */ +class AggValues { + private int row = 0; + + private final List columns; + private int[] indexPerLevel; + private int size; + + AggValues(List columns) { + this.columns = columns; + } + + void init(int maxDepth, int limit) { + int sz = computeSize(columns, maxDepth); + size = limit > 0 ? Math.min(limit, sz) : sz; + indexPerLevel = new int[maxDepth + 1]; + } + + private static int computeSize(List columns, int maxDepth) { + // look only at arrays with the right depth (the others might be + // counters or other functions) + // then return the parent array to compute the actual returned results + Object[] leafArray = null; + for (int i = 0; i < columns.size() && leafArray == null; i++) { + Object[] col = columns.get(i); + Object o = col; + int level = 0; + Object[] parent = null; + // keep unwrapping until the desired level is reached + while (o instanceof Object[]) { + col = ((Object[]) o); + if (col.length > 0) { + if (level == maxDepth) { + leafArray = parent; + break; + } else { + parent = col; + level++; + o = col[0]; + } + } else { + o = null; + } + } + } + + if (leafArray == null) { + return columns.get(0).length; + } + + int sz = 0; + for (Object leaf : leafArray) { + sz += ((Object[]) leaf).length; + } + return sz; + } + + Object column(int column) { + Object o = columns.get(column); + + for (int lvl = 0; o instanceof Object[]; lvl++) { + Object[] arr = (Object[]) o; + // the current branch is done + if (indexPerLevel[lvl] == arr.length) { + // reset the current branch + indexPerLevel[lvl] = 0; + // bump the parent - if it's too big it, the loop will restart + // again from that position + indexPerLevel[lvl - 1]++; + // restart the loop + lvl = -1; + o = columns.get(column); + } else { + o = arr[indexPerLevel[lvl]]; + } + } + return o; + } + + int size() { + return size; + } + + void reset() { + row = 0; + Arrays.fill(indexPerLevel, 0); + } + + boolean nextRow() { + if (row < size - 1) { + row++; + // increment leaf counter - the size check is done lazily while retrieving the columns + indexPerLevel[indexPerLevel.length - 1]++; + return true; + } + return false; + } + + boolean hasCurrentRow() { + return row < size; + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/AggsRowSetCursor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/AggsRowSetCursor.java index 35475c90199..32507d7686c 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/AggsRowSetCursor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/AggsRowSetCursor.java @@ -5,106 +5,28 @@ */ package org.elasticsearch.xpack.sql.execution.search; -import java.util.Arrays; -import java.util.List; - -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.xpack.sql.session.AbstractRowSetCursor; import org.elasticsearch.xpack.sql.session.Cursor; import org.elasticsearch.xpack.sql.type.Schema; -// -// Aggregations are returned in a tree structure where each nested level can have a different size. -// For example a group by a, b, c results in 3-level nested array where each level contains all the relevant values -// for its parent entry. -// Assuming there's a total of 2 A's, 3 B's and 5 C's, the values will be -// A-agg level = { A1, A2 } -// B-agg level = { { A1B1, A1B2, A1B3 }, { A2B1, A2B2, A2B3 } -// C-agg level = { { { A1B1C1, A1B1C2 ..}, { A1B2C1, etc... } } } and so on -// -// To help with the iteration, there are two dedicated counters : -// - one that carries (increments) the counter for each level (indicated by the position inside the array) once the children reach their max -// - a flat cursor to indicate the row +import java.util.List; +import java.util.function.Supplier; class AggsRowSetCursor extends AbstractRowSetCursor { private int row = 0; + private final AggValues agg; + private final List> columns; - private final List columns; - private final int[] indexPerLevel; - private final int size; - - AggsRowSetCursor(Schema schema, List columns, int maxDepth, int limit) { + AggsRowSetCursor(Schema schema, AggValues agg, List> columns) { super(schema, null); + this.agg = agg; this.columns = columns; - - int sz = computeSize(columns, maxDepth); - size = limit > 0 ? Math.min(limit, sz) : sz; - indexPerLevel = new int[maxDepth + 1]; - } - - private static int computeSize(List columns, int maxDepth) { - // look only at arrays with the right depth (the others might be counters or other functions) - // then return the parent array to compute the actual returned results - Object[] leafArray = null; - for (int i = 0; i < columns.size() && leafArray == null; i++) { - Object[] col = columns.get(i); - Object o = col; - int level = 0; - Object[] parent = null; - // keep unwrapping until the desired level is reached - while (o instanceof Object[]) { - col = ((Object[]) o); - if (col.length > 0) { - if (level == maxDepth) { - leafArray = parent; - break; - } - else { - parent = col; - level++; - o = col[0]; - } - } - else { - o = null; - } - } - } - - if (leafArray == null) { - return columns.get(0).length; - } - - int sz = 0; - for (Object leaf : leafArray) { - sz += ((Object[]) leaf).length; - } - return sz; } @Override protected Object getColumn(int column) { - Object o = columns.get(column); - - for (int lvl = 0; o instanceof Object[]; lvl++) { - Object[] arr = (Object[]) o; - // the current branch is done - if (indexPerLevel[lvl] == arr.length) { - // reset the current branch - indexPerLevel[lvl] = 0; - // bump the parent - if it's too big it, the loop will restart again from that position - indexPerLevel[lvl - 1]++; - // restart the loop - lvl = -1; - o = columns.get(column); - } - else { - o = arr[indexPerLevel[lvl]]; - } - } - return o; + return columns.get(column).get(); } @Override @@ -114,24 +36,17 @@ class AggsRowSetCursor extends AbstractRowSetCursor { @Override protected boolean doNext() { - if (row < size() - 1) { - row++; - // increment leaf counter - the size check is done lazily while retrieving the columns - indexPerLevel[indexPerLevel.length - 1]++; - return true; - } - return false; + return agg.nextRow(); } @Override protected void doReset() { - row = 0; - Arrays.fill(indexPerLevel, 0); + agg.reset(); } @Override public int size() { - return size; + return agg.size(); } @Override diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/ProcessingHitExtractor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/ProcessingHitExtractor.java deleted file mode 100644 index 5ac3ad5b8c9..00000000000 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/ProcessingHitExtractor.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.sql.execution.search; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.xpack.sql.expression.function.scalar.ColumnProcessor; - -import java.io.IOException; -import java.util.Objects; - -class ProcessingHitExtractor implements HitExtractor { - static final String NAME = "p"; - private final HitExtractor delegate; - private final ColumnProcessor processor; - - ProcessingHitExtractor(HitExtractor delegate, ColumnProcessor processor) { - this.delegate = delegate; - this.processor = processor; - } - - ProcessingHitExtractor(StreamInput in) throws IOException { - delegate = in.readNamedWriteable(HitExtractor.class); - processor = in.readNamedWriteable(ColumnProcessor.class); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeNamedWriteable(delegate); - out.writeNamedWriteable(processor); - } - - @Override - public String getWriteableName() { - return NAME; - } - - HitExtractor delegate() { - return delegate; - } - - ColumnProcessor processor() { - return processor; - } - - @Override - public Object get(SearchHit hit) { - return processor.apply(delegate.get(hit)); - } - - @Override - public String innerHitName() { - return delegate.innerHitName(); - } - - @Override - public boolean equals(Object obj) { - if (obj == null || obj.getClass() != getClass()) { - return false; - } - ProcessingHitExtractor other = (ProcessingHitExtractor) obj; - return delegate.equals(other.delegate) - && processor.equals(other.processor); - } - - @Override - public int hashCode() { - return Objects.hash(delegate, processor); - } - - @Override - public String toString() { - return processor + "(" + delegate + ")"; - } -} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/ScrollCursor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/ScrollCursor.java index fddd6450269..cc4a5b1872b 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/ScrollCursor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/ScrollCursor.java @@ -16,6 +16,8 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.sql.execution.search.extractor.HitExtractor; +import org.elasticsearch.xpack.sql.execution.search.extractor.HitExtractors; import org.elasticsearch.xpack.sql.session.Cursor; import org.elasticsearch.xpack.sql.session.RowSetCursor; import org.elasticsearch.xpack.sql.type.DataType; @@ -36,7 +38,7 @@ public class ScrollCursor implements Cursor { /** * {@link NamedWriteableRegistry} used to resolve the {@link #extractors}. */ - private static final NamedWriteableRegistry REGISTRY = new NamedWriteableRegistry(HitExtractor.getNamedWriteables()); + private static final NamedWriteableRegistry REGISTRY = new NamedWriteableRegistry(HitExtractors.getNamedWriteables()); private final String scrollId; private final List extractors; diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/Scroller.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/Scroller.java index 8faee2868d1..35a987c3db3 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/Scroller.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/Scroller.java @@ -23,13 +23,24 @@ import org.elasticsearch.search.aggregations.support.AggregationPath; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.execution.ExecutionException; -import org.elasticsearch.xpack.sql.expression.function.scalar.ColumnProcessor; +import org.elasticsearch.xpack.sql.execution.search.extractor.ComputingHitExtractor; +import org.elasticsearch.xpack.sql.execution.search.extractor.ConstantExtractor; +import org.elasticsearch.xpack.sql.execution.search.extractor.DocValueExtractor; +import org.elasticsearch.xpack.sql.execution.search.extractor.HitExtractor; +import org.elasticsearch.xpack.sql.execution.search.extractor.InnerHitExtractor; +import org.elasticsearch.xpack.sql.execution.search.extractor.SourceExtractor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.AggPathInput; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.AggValueInput; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.HitExtractorInput; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ReferenceInput; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; import org.elasticsearch.xpack.sql.querydsl.agg.AggPath; import org.elasticsearch.xpack.sql.querydsl.container.AggRef; +import org.elasticsearch.xpack.sql.querydsl.container.ColumnReference; +import org.elasticsearch.xpack.sql.querydsl.container.ComputedRef; import org.elasticsearch.xpack.sql.querydsl.container.NestedFieldRef; -import org.elasticsearch.xpack.sql.querydsl.container.ProcessingRef; import org.elasticsearch.xpack.sql.querydsl.container.QueryContainer; -import org.elasticsearch.xpack.sql.querydsl.container.Reference; import org.elasticsearch.xpack.sql.querydsl.container.ScriptFieldRef; import org.elasticsearch.xpack.sql.querydsl.container.SearchHitFieldRef; import org.elasticsearch.xpack.sql.querydsl.container.TotalCountRef; @@ -38,11 +49,12 @@ import org.elasticsearch.xpack.sql.session.Rows; import org.elasticsearch.xpack.sql.session.SqlSettings; import org.elasticsearch.xpack.sql.type.Schema; import org.elasticsearch.xpack.sql.util.ObjectUtils; +import org.elasticsearch.xpack.sql.util.StringUtils; import java.util.ArrayList; import java.util.List; import java.util.function.Consumer; - +import java.util.function.Supplier; // TODO: add retry/back-off public class Scroller { @@ -68,7 +80,9 @@ public class Scroller { // prepare the request SearchSourceBuilder sourceBuilder = SourceGenerator.sourceBuilder(query); - log.trace("About to execute query {} on {}", sourceBuilder, index); + if (log.isTraceEnabled()) { + log.trace("About to execute query {} on {}", StringUtils.toString(sourceBuilder), index); + } SearchRequest search = client.prepareSearch(index).setSource(sourceBuilder).request(); search.scroll(keepAlive).source().timeout(timeout); @@ -79,7 +93,9 @@ public class Scroller { search.source().size(sz); } - ScrollerActionListener l = query.isAggsOnly() ? new AggsScrollActionListener(listener, client, timeout, schema, query) : new HandshakeScrollActionListener(listener, client, timeout, schema, query); + boolean isAggsOnly = query.isAggsOnly(); + + ScrollerActionListener l = isAggsOnly ? new AggsScrollActionListener(listener, client, timeout, schema, query) : new HandshakeScrollActionListener(listener, client, timeout, schema, query); client.search(search, l); } @@ -91,7 +107,7 @@ public class Scroller { // dedicated scroll used for aggs-only/group-by results static class AggsScrollActionListener extends ScrollerActionListener { - + private final QueryContainer query; AggsScrollActionListener(ActionListener listener, Client client, TimeValue keepAlive, Schema schema, QueryContainer query) { @@ -101,72 +117,90 @@ public class Scroller { @Override protected RowSetCursor handleResponse(SearchResponse response) { - Aggregations aggs = response.getAggregations(); - - List columns = new ArrayList<>(); + + final List extractedAggs = new ArrayList<>(); + AggValues aggValues = new AggValues(extractedAggs); + List> aggColumns = new ArrayList<>(query.columns().size()); // this method assumes the nested aggregation are all part of the same tree (the SQL group-by) int maxDepth = -1; + + List cols = query.columns(); + for (int index = 0; index < cols.size(); index++) { + ColumnReference col = cols.get(index); + Supplier supplier = null; - for (Reference ref : query.refs()) { - Object[] arr = null; - - ColumnProcessor processor = null; - - if (ref instanceof ProcessingRef) { - ProcessingRef pRef = (ProcessingRef) ref; - processor = pRef.processor(); - ref = pRef.ref(); + if (col instanceof ComputedRef) { + ComputedRef pRef = (ComputedRef) col; + + Processor processor = pRef.processor().transformUp(a -> { + Object[] value = extractAggValue(new AggRef(a.context()), response); + extractedAggs.add(value); + final int aggPosition = extractedAggs.size() - 1; + return new AggValueInput(a.expression(), () -> aggValues.column(aggPosition), a.innerKey()); + }, AggPathInput.class).asProcessor(); + // the input is provided through the value input above + supplier = () -> processor.process(null); + } + else { + extractedAggs.add(extractAggValue(col, response)); + final int aggPosition = extractedAggs.size() - 1; + supplier = () -> aggValues.column(aggPosition); } - if (ref == TotalCountRef.INSTANCE) { - arr = new Object[] { processIfNeeded(processor, Long.valueOf(response.getHits().getTotalHits())) }; - columns.add(arr); + aggColumns.add(supplier); + if (col.depth() > maxDepth) { + maxDepth = col.depth(); } - else if (ref instanceof AggRef) { + } + + aggValues.init(maxDepth, query.limit()); + clearScroll(response.getScrollId()); + + return new AggsRowSetCursor(schema, aggValues, aggColumns); + } + + private Object[] extractAggValue(ColumnReference col, SearchResponse response) { + if (col == TotalCountRef.INSTANCE) { + return new Object[] { Long.valueOf(response.getHits().getTotalHits()) }; + } + else if (col instanceof AggRef) { + Object[] arr; + + String path = ((AggRef) col).path(); + // yup, this is instance equality to make sure we only check the path used by the code + if (path == TotalCountRef.PATH) { + arr = new Object[] { Long.valueOf(response.getHits().getTotalHits()) }; + } + else { // workaround for elastic/elasticsearch/issues/23056 - String path = ((AggRef) ref).path(); boolean formattedKey = AggPath.isBucketValueFormatted(path); if (formattedKey) { path = AggPath.bucketValueWithoutFormat(path); } - Object value = getAggProperty(aggs, path); - - // // FIXME: this can be tabular in nature - // if (ref instanceof MappedAggRef) { - // Map map = (Map) value; - // Object extractedValue = map.get(((MappedAggRef) ref).fieldName()); - // } - + Object value = getAggProperty(response.getAggregations(), path); + + // // FIXME: this can be tabular in nature + // if (ref instanceof MappedAggRef) { + // Map map = (Map) value; + // Object extractedValue = map.get(((MappedAggRef) + // ref).fieldName()); + // } + if (formattedKey) { List buckets = ((MultiBucketsAggregation) value).getBuckets(); arr = new Object[buckets.size()]; for (int i = 0; i < buckets.size(); i++) { arr[i] = buckets.get(i).getKeyAsString(); } - } - else { + } else { arr = value instanceof Object[] ? (Object[]) value : new Object[] { value }; } - - // process if needed - for (int i = 0; i < arr.length; i++) { - arr[i] = processIfNeeded(processor, arr[i]); - } - columns.add(arr); - } - // aggs without any grouping - else { - throw new SqlIllegalArgumentException("Unexpected non-agg/grouped column specified; %s", ref.getClass()); - } - - if (ref.depth() > maxDepth) { - maxDepth = ref.depth(); } + + return arr; } - - clearScroll(response.getScrollId()); - return new AggsRowSetCursor(schema, columns, maxDepth, query.limit()); + throw new SqlIllegalArgumentException("Unexpected non-agg/grouped column specified; %s", col.getClass()); } private static Object getAggProperty(Aggregations aggs, String path) { @@ -178,10 +212,6 @@ public class Scroller { } return agg.getProperty(list.subList(1, list.size())); } - - private Object processIfNeeded(ColumnProcessor processor, Object value) { - return processor != null ? processor.apply(value) : value; - } } // initial scroll used for parsing search hits (handles possible aggs) @@ -202,17 +232,17 @@ public class Scroller { @Override protected List getExtractors() { // create response extractors for the first time - List refs = query.refs(); + List refs = query.columns(); List exts = new ArrayList<>(refs.size()); - for (Reference ref : refs) { + for (ColumnReference ref : refs) { exts.add(createExtractor(ref)); } return exts; } - private HitExtractor createExtractor(Reference ref) { + private HitExtractor createExtractor(ColumnReference ref) { if (ref instanceof SearchHitFieldRef) { SearchHitFieldRef f = (SearchHitFieldRef) ref; return f.useDocValue() ? new DocValueExtractor(f.name()) : new SourceExtractor(f.name()); @@ -228,9 +258,10 @@ public class Scroller { return new DocValueExtractor(f.name()); } - if (ref instanceof ProcessingRef) { - ProcessingRef pRef = (ProcessingRef) ref; - return new ProcessingHitExtractor(createExtractor(pRef.ref()), pRef.processor()); + if (ref instanceof ComputedRef) { + ProcessorDefinition proc = ((ComputedRef) ref).processor(); + proc = proc.transformDown(l -> new HitExtractorInput(l.expression(), createExtractor(l.context())), ReferenceInput.class); + return new ComputingHitExtractor(proc.asProcessor()); } throw new SqlIllegalArgumentException("Unexpected ValueReference %s", ref.getClass()); @@ -303,7 +334,8 @@ public class Scroller { private static boolean needsHit(List exts) { for (HitExtractor ext : exts) { - if (ext instanceof DocValueExtractor || ext instanceof ProcessingHitExtractor) { + // Anything non-constant requires extraction + if (!(ext instanceof ConstantExtractor)) { return true; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/SearchHitRowSetCursor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/SearchHitRowSetCursor.java index 6612b233747..a5c1d3a2b76 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/SearchHitRowSetCursor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/SearchHitRowSetCursor.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; +import org.elasticsearch.xpack.sql.execution.search.extractor.HitExtractor; import org.elasticsearch.xpack.sql.session.AbstractRowSetCursor; import org.elasticsearch.xpack.sql.session.Cursor; import org.elasticsearch.xpack.sql.session.RowSetCursor; diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java index 03de4cdb470..5bbdbd9d774 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java @@ -5,12 +5,8 @@ */ package org.elasticsearch.xpack.sql.execution.search; -import java.util.ArrayList; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; - import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.script.Script; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -18,27 +14,36 @@ import org.elasticsearch.search.fetch.StoredFieldsContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.expression.Attribute; import org.elasticsearch.xpack.sql.expression.FieldAttribute; import org.elasticsearch.xpack.sql.expression.NestedFieldAttribute; import org.elasticsearch.xpack.sql.expression.RootFieldAttribute; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ReferenceInput; import org.elasticsearch.xpack.sql.querydsl.agg.Aggs; import org.elasticsearch.xpack.sql.querydsl.container.AttributeSort; -import org.elasticsearch.xpack.sql.querydsl.container.ProcessingRef; +import org.elasticsearch.xpack.sql.querydsl.container.ColumnReference; +import org.elasticsearch.xpack.sql.querydsl.container.ComputedRef; import org.elasticsearch.xpack.sql.querydsl.container.QueryContainer; -import org.elasticsearch.xpack.sql.querydsl.container.Reference; import org.elasticsearch.xpack.sql.querydsl.container.ScriptFieldRef; import org.elasticsearch.xpack.sql.querydsl.container.ScriptSort; import org.elasticsearch.xpack.sql.querydsl.container.SearchHitFieldRef; import org.elasticsearch.xpack.sql.querydsl.container.Sort; import org.elasticsearch.xpack.sql.querydsl.container.Sort.Direction; import org.elasticsearch.xpack.sql.querydsl.query.NestedQuery; -import org.elasticsearch.search.sort.SortBuilder; -import org.elasticsearch.search.sort.SortOrder; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; import static java.util.Collections.singletonList; - import static org.elasticsearch.search.sort.SortBuilders.fieldSort; import static org.elasticsearch.search.sort.SortBuilders.scriptSort; @@ -56,29 +61,20 @@ public abstract class SourceGenerator { // translate fields to source-fields or script fields Set sourceFields = new LinkedHashSet<>(); Set docFields = new LinkedHashSet<>(); - for (Reference ref : container.refs()) { - if (ref instanceof ProcessingRef) { - ref = ((ProcessingRef) ref).ref(); - } + Map scriptFields = new LinkedHashMap<>(); - if (ref instanceof SearchHitFieldRef) { - SearchHitFieldRef sh = (SearchHitFieldRef) ref; - Set collection = sh.useDocValue() ? docFields : sourceFields; - collection.add(ref.toString()); - } - else if (ref instanceof ScriptFieldRef) { - ScriptFieldRef sfr = (ScriptFieldRef) ref; - source.scriptField(sfr.name(), sfr.script().toPainless()); - } + for (ColumnReference ref : container.columns()) { + collectFields(ref, sourceFields, docFields, scriptFields); } if (!sourceFields.isEmpty()) { source.fetchSource(sourceFields.toArray(new String[sourceFields.size()]), null); } - if (!docFields.isEmpty()) { - for (String field : docFields) { - source.docValueField(field); - } + for (String field : docFields) { + source.docValueField(field); + } + for (Entry entry : scriptFields.entrySet()) { + source.scriptField(entry.getKey(), entry.getValue()); } sorting(container, source); @@ -99,6 +95,22 @@ public abstract class SourceGenerator { return source; } + private static void collectFields(ColumnReference ref, Set sourceFields, Set docFields, Map scriptFields) { + if (ref instanceof ComputedRef) { + ProcessorDefinition proc = ((ComputedRef) ref).processor(); + proc.forEachUp(l -> collectFields(l.context(), sourceFields, docFields, scriptFields), ReferenceInput.class); + } + else if (ref instanceof SearchHitFieldRef) { + SearchHitFieldRef sh = (SearchHitFieldRef) ref; + Set collection = sh.useDocValue() ? docFields : sourceFields; + collection.add(sh.name()); + } + else if (ref instanceof ScriptFieldRef) { + ScriptFieldRef sfr = (ScriptFieldRef) ref; + scriptFields.put(sfr.name(), sfr.script().toPainless()); + } + } + private static void sorting(QueryContainer container, SearchSourceBuilder source) { if (container.sort() != null) { diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingHitExtractor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingHitExtractor.java new file mode 100644 index 00000000000..c64aa6368f5 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingHitExtractor.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.execution.search.extractor; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.HitExtractorProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; + +import java.io.IOException; +import java.util.Objects; + +/** + * HitExtractor that delegates to a processor. The difference between this class + * and {@link HitExtractorProcessor} is that the latter is used inside a + * {@link Processor} tree as a leaf (and thus can effectively parse the + * {@link SearchHit} while this class is used when scrolling and passing down + * the results. + * + * In the future, the processor might be used across the board for all columns + * to reduce API complexity (and keep the {@link HitExtractor} only as an + * internal implementation detail). + */ +public class ComputingHitExtractor implements HitExtractor { + static final String NAME = "p"; + private final Processor processor; + + public ComputingHitExtractor(Processor processor) { + this.processor = processor; + } + + ComputingHitExtractor(StreamInput in) throws IOException { + processor = in.readNamedWriteable(Processor.class); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeNamedWriteable(processor); + } + + @Override + public String getWriteableName() { + return NAME; + } + + public Processor processor() { + return processor; + } + + @Override + public Object get(SearchHit hit) { + return processor.process(hit); + } + + @Override + public String innerHitName() { + return null; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + ComputingHitExtractor other = (ComputingHitExtractor) obj; + return processor.equals(other.processor); + } + + @Override + public int hashCode() { + return Objects.hash(processor); + } + + @Override + public String toString() { + return processor.toString(); + } +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/ConstantExtractor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ConstantExtractor.java similarity index 90% rename from sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/ConstantExtractor.java rename to sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ConstantExtractor.java index cdffbe68072..b0112c91e9e 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/ConstantExtractor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/ConstantExtractor.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.sql.execution.search; +package org.elasticsearch.xpack.sql.execution.search.extractor; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -15,11 +15,11 @@ import java.util.Objects; /** * Returns the a constant for every search hit against which it is run. */ -class ConstantExtractor implements HitExtractor { +public class ConstantExtractor implements HitExtractor { static final String NAME = "c"; private final Object constant; - ConstantExtractor(Object constant) { + public ConstantExtractor(Object constant) { this.constant = constant; } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/DocValueExtractor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/DocValueExtractor.java similarity index 91% rename from sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/DocValueExtractor.java rename to sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/DocValueExtractor.java index a1588a9fc7c..11ec2266bfe 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/DocValueExtractor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/DocValueExtractor.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.sql.execution.search; +package org.elasticsearch.xpack.sql.execution.search.extractor; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.io.stream.StreamInput; @@ -15,11 +15,11 @@ import java.io.IOException; /** * Extracts field values from {@link SearchHit#field(String)}. */ -class DocValueExtractor implements HitExtractor { +public class DocValueExtractor implements HitExtractor { static final String NAME = "f"; private final String fieldName; - DocValueExtractor(String name) { + public DocValueExtractor(String name) { this.fieldName = name; } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/HitExtractor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/HitExtractor.java new file mode 100644 index 00000000000..a3936b9fe16 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/HitExtractor.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.execution.search.extractor; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.search.SearchHit; + +/** + * Extracts a column value from a {@link SearchHit}. + */ +public interface HitExtractor extends NamedWriteable { + /** + * Extract the value from a hit. + */ + Object get(SearchHit hit); + + /** + * Name of the inner hit needed by this extractor if it needs one, {@code null} otherwise. + */ + @Nullable + String innerHitName(); +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/HitExtractor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/HitExtractors.java similarity index 52% rename from sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/HitExtractor.java rename to sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/HitExtractors.java index de06d9dc39f..729b951a831 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/HitExtractor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/HitExtractors.java @@ -3,45 +3,29 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.sql.execution.search; +package org.elasticsearch.xpack.sql.execution.search.extractor; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.xpack.sql.expression.function.scalar.ColumnProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.Processors; import java.util.ArrayList; import java.util.List; -/** - * Extracts a columns value from a {@link SearchHit}. - */ -public interface HitExtractor extends NamedWriteable { +public abstract class HitExtractors { + /** - * All of the named writeables needed to deserialize the instances - * of {@linkplain HitExtractor}. + * All of the named writeables needed to deserialize the instances of + * {@linkplain HitExtractor}. */ - static List getNamedWriteables() { + public static List getNamedWriteables() { List entries = new ArrayList<>(); entries.add(new Entry(HitExtractor.class, ConstantExtractor.NAME, ConstantExtractor::new)); entries.add(new Entry(HitExtractor.class, DocValueExtractor.NAME, DocValueExtractor::new)); entries.add(new Entry(HitExtractor.class, InnerHitExtractor.NAME, InnerHitExtractor::new)); entries.add(new Entry(HitExtractor.class, SourceExtractor.NAME, SourceExtractor::new)); - entries.add(new Entry(HitExtractor.class, ProcessingHitExtractor.NAME, ProcessingHitExtractor::new)); - entries.addAll(ColumnProcessor.getNamedWriteables()); + entries.add(new Entry(HitExtractor.class, ComputingHitExtractor.NAME, ComputingHitExtractor::new)); + entries.addAll(Processors.getNamedWriteables()); return entries; } - - /** - * Extract the value from a hit. - */ - Object get(SearchHit hit); - - /** - * Name of the inner hit needed by this extractor if it needs one, {@code null} otherwise. - */ - @Nullable - String innerHitName(); -} \ No newline at end of file +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/InnerHitExtractor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/InnerHitExtractor.java similarity index 94% rename from sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/InnerHitExtractor.java rename to sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/InnerHitExtractor.java index 16b169e43c2..7d90659d979 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/InnerHitExtractor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/InnerHitExtractor.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.sql.execution.search; +package org.elasticsearch.xpack.sql.execution.search.extractor; import org.elasticsearch.common.Strings; import org.elasticsearch.common.document.DocumentField; @@ -16,13 +16,13 @@ import java.io.IOException; import java.util.Map; import java.util.Objects; -class InnerHitExtractor implements HitExtractor { +public class InnerHitExtractor implements HitExtractor { static final String NAME = "i"; private final String hitName, fieldName; private final boolean useDocValue; private final String[] tree; - InnerHitExtractor(String hitName, String name, boolean useDocValue) { + public InnerHitExtractor(String hitName, String name, boolean useDocValue) { this.hitName = hitName; this.fieldName = name; this.useDocValue = useDocValue; diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceExtractor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/SourceExtractor.java similarity index 92% rename from sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceExtractor.java rename to sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/SourceExtractor.java index 218032be2e6..94313bb2450 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceExtractor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/SourceExtractor.java @@ -3,20 +3,20 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.sql.execution.search; - -import java.io.IOException; -import java.util.Map; +package org.elasticsearch.xpack.sql.execution.search.extractor; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.search.SearchHit; -class SourceExtractor implements HitExtractor { +import java.io.IOException; +import java.util.Map; + +public class SourceExtractor implements HitExtractor { public static final String NAME = "s"; private final String fieldName; - SourceExtractor(String name) { + public SourceExtractor(String name) { this.fieldName = name; } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/BinaryExpression.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/BinaryExpression.java index ed3a82e510f..0b654dd26d4 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/BinaryExpression.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/BinaryExpression.java @@ -5,21 +5,15 @@ */ package org.elasticsearch.xpack.sql.expression; +import org.elasticsearch.xpack.sql.tree.Location; + import java.util.Arrays; import java.util.Objects; -import org.elasticsearch.xpack.sql.tree.Location; -import org.elasticsearch.xpack.sql.type.DataType; -import org.elasticsearch.xpack.sql.type.DataTypes; - public abstract class BinaryExpression extends Expression { private final Expression left, right; - public interface Negateable { - BinaryExpression negate(); - } - protected BinaryExpression(Location location, Expression left, Expression right) { super(location, Arrays.asList(left, right)); this.left = left; @@ -44,13 +38,6 @@ public abstract class BinaryExpression extends Expression { return left.nullable() || left.nullable(); } - public abstract BinaryExpression swapLeftAndRight(); - - @Override - public DataType dataType() { - return DataTypes.BOOLEAN; - } - @Override public int hashCode() { return Objects.hash(left, right); @@ -79,6 +66,7 @@ public abstract class BinaryExpression extends Expression { return sb.toString(); } - // simplify toString public abstract String symbol(); + + public abstract BinaryExpression swapLeftAndRight(); } \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/BinaryLogic.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/BinaryLogic.java new file mode 100644 index 00000000000..edef746e41c --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/BinaryLogic.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression; + +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DataTypes; + +public abstract class BinaryLogic extends BinaryOperator { + + protected BinaryLogic(Location location, Expression left, Expression right) { + super(location, left, right); + } + + @Override + public DataType dataType() { + return DataTypes.BOOLEAN; + } + + @Override + protected TypeResolution resolveInputType(DataType inputType) { + return DataTypes.BOOLEAN.equals(inputType) ? TypeResolution.TYPE_RESOLVED : new TypeResolution( + "'%s' requires type %s not %s", symbol(), DataTypes.BOOLEAN.sqlName(), inputType.sqlName()); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/BinaryOperator.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/BinaryOperator.java index 34677dd46c9..8390a86c4ad 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/BinaryOperator.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/BinaryOperator.java @@ -7,30 +7,33 @@ package org.elasticsearch.xpack.sql.expression; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.type.DataType; -import org.elasticsearch.xpack.sql.type.DataTypeConversion; - +//Binary expression that requires both input expressions to have the same type +//Compatible types should be handled by the analyzer (by using the narrowest type) public abstract class BinaryOperator extends BinaryExpression { + public interface Negateable { + BinaryExpression negate(); + } + protected BinaryOperator(Location location, Expression left, Expression right) { super(location, left, right); } - protected abstract DataType acceptedType(); + protected abstract TypeResolution resolveInputType(DataType inputType); @Override protected TypeResolution resolveType() { - DataType accepted = acceptedType(); + if (!childrenResolved()) { + return new TypeResolution("Unresolved children"); + } DataType l = left().dataType(); DataType r = right().dataType(); - if (!l.same(r)) { - return new TypeResolution("Different types (%s and %s) used in '%s'", l.sqlName(), r.sqlName(), symbol()); - } - if (!DataTypeConversion.canConvert(accepted, left().dataType())) { - return new TypeResolution("'%s' requires type %s not %s", symbol(), accepted.sqlName(), l.sqlName()); - } - else { - return TypeResolution.TYPE_RESOLVED; + TypeResolution resolution = resolveInputType(l); + + if (resolution == TypeResolution.TYPE_RESOLVED) { + return resolveInputType(r); } + return resolution; } -} +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/Expression.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/Expression.java index 46504697fee..3a839c11f0f 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/Expression.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/Expression.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.sql.expression; +import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.capabilities.Resolvable; import org.elasticsearch.xpack.sql.capabilities.Resolvables; import org.elasticsearch.xpack.sql.tree.Location; @@ -62,7 +63,7 @@ public abstract class Expression extends Node implements Resolvable } public Object fold() { - return null; + throw new SqlIllegalArgumentException("Should not fold expression"); } public abstract boolean nullable(); diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java index 6aa02ca802c..9cb20dfdc7a 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.sql.expression; +import org.elasticsearch.xpack.sql.expression.Expression.TypeResolution; + import java.util.ArrayList; import java.util.List; import java.util.function.Predicate; @@ -68,4 +70,14 @@ public abstract class Expressions { public static Attribute attribute(Expression e) { return e instanceof NamedExpression ? ((NamedExpression) e).toAttribute() : null; } + + public static TypeResolution typeMustBe(Expression e, Predicate predicate, String message) { + return predicate.test(e) ? TypeResolution.TYPE_RESOLVED : new TypeResolution(message); + } + + public static TypeResolution typeMustBeNumeric(Expression e) { + return e.dataType().isNumeric()? TypeResolution.TYPE_RESOLVED : new TypeResolution( + "Argument required to be numeric ('%s' of type '%s')", Expressions.name(e), e.dataType().esName()); + } + } \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/Literal.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/Literal.java index 4d9e1d87ce3..3488451be68 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/Literal.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/Literal.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.sql.expression; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DataTypeConversion; import org.elasticsearch.xpack.sql.type.DataTypes; import java.util.Objects; @@ -21,8 +22,8 @@ public class Literal extends LeafExpression { public Literal(Location location, Object value, DataType dataType) { super(location); - this.value = value; this.dataType = dataType; + this.value = DataTypeConversion.convert(value, dataType); } public Object value() { diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/UnaryExpression.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/UnaryExpression.java index 68e0ee249dd..40c264e9bb9 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/UnaryExpression.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/UnaryExpression.java @@ -30,11 +30,6 @@ public abstract class UnaryExpression extends Expression { return child.nullable(); } - @Override - public boolean foldable() { - return child.foldable(); - } - @Override public boolean resolved() { return child.resolved(); diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/Function.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/Function.java index 5f4b563cf57..1b16ec506e7 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/Function.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/Function.java @@ -40,11 +40,6 @@ public abstract class Function extends NamedExpression { return name; } - @Override - public boolean foldable() { - return false; - } - @Override public boolean nullable() { return false; diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/Functions.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/Functions.java index 578e5fb62e8..ed55bb0a1e4 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/Functions.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/Functions.java @@ -9,15 +9,8 @@ import org.elasticsearch.xpack.sql.expression.Alias; import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.expression.NamedExpression; import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunction; -import org.elasticsearch.xpack.sql.expression.function.scalar.ColumnProcessor; -import org.elasticsearch.xpack.sql.expression.function.scalar.ComposeProcessor; -import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction; - -import java.util.ArrayList; -import java.util.List; - -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; +import org.elasticsearch.xpack.sql.expression.function.scalar.BinaryScalarFunction; +import org.elasticsearch.xpack.sql.expression.function.scalar.UnaryScalarFunction; public abstract class Functions { @@ -25,8 +18,11 @@ public abstract class Functions { return e instanceof AggregateFunction; } - public static boolean isScalarFunction(Expression e) { - return e instanceof ScalarFunction; + public static boolean isUnaryScalarFunction(Expression e) { + if (e instanceof BinaryScalarFunction) { + throw new UnsupportedOperationException("not handled currently"); + } + return e instanceof UnaryScalarFunction; } public static AggregateFunction extractAggregate(NamedExpression ne) { @@ -35,8 +31,11 @@ public abstract class Functions { if (e instanceof Alias) { e = ((Alias) ne).child(); } - else if (e instanceof ScalarFunction) { - e = ((ScalarFunction) e).argument(); + else if (e instanceof UnaryScalarFunction) { + e = ((UnaryScalarFunction) e).field(); + } + else if (e instanceof BinaryScalarFunction) { + throw new UnsupportedOperationException(); } else if (e instanceof AggregateFunction) { return (AggregateFunction) e; @@ -47,52 +46,4 @@ public abstract class Functions { } return null; } - - public static List unwrapScalarFunctionWithTail(Expression e) { - if (!(e instanceof ScalarFunction)) { - return emptyList(); - } - List exps = new ArrayList<>(); - while (isScalarFunction(e)) { - ScalarFunction scalar = (ScalarFunction) e; - exps.add(scalar); - e = scalar.argument(); - } - exps.add(e); - return exps; - } - - public static List unwrapScalarProcessor(Expression e) { - if (!(e instanceof ScalarFunction)) { - return emptyList(); - } - - // common-case (single function wrapper) - if (e instanceof ScalarFunction && !(((ScalarFunction) e).argument() instanceof ScalarFunction)) { - return singletonList((ScalarFunction) e); - } - - List exps = new ArrayList<>(); - while (e instanceof ScalarFunction) { - ScalarFunction scalar = (ScalarFunction) e; - exps.add(scalar); - e = scalar.argument(); - } - return exps; - } - - public static ColumnProcessor chainProcessors(List unwrappedScalar) { - ColumnProcessor proc = null; - for (Expression e : unwrappedScalar) { - if (e instanceof ScalarFunction) { - ScalarFunction sf = (ScalarFunction) e; - // A(B(C)) is applied backwards first C then B then A, the last function first - proc = proc == null ? sf.asProcessor() : new ComposeProcessor(sf.asProcessor(), proc); - } - else { - return proc; - } - } - return proc; - } } \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Avg.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Avg.java index 64a56adeb33..8c63dcdebf2 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Avg.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Avg.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.sql.expression.function.aggregate; import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.type.DataType; public class Avg extends NumericAggregate implements EnclosedAgg { @@ -18,4 +19,9 @@ public class Avg extends NumericAggregate implements EnclosedAgg { public String innerName() { return "avg"; } + + @Override + public DataType dataType() { + return field().dataType(); + } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/NumericAggregate.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/NumericAggregate.java index d25d6fe27a1..09306c35a07 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/NumericAggregate.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/NumericAggregate.java @@ -25,10 +25,7 @@ class NumericAggregate extends AggregateFunction { @Override protected TypeResolution resolveType() { - return field().dataType().isNumeric() ? TypeResolution.TYPE_RESOLVED : new TypeResolution( - "Function '%s' cannot be applied on a non-numeric expression ('%s' of type '%s')", functionName(), - Expressions.name(field()), field().dataType().esName()); - + return Expressions.typeMustBeNumeric(field()); } @Override diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Percentile.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Percentile.java index 96b9207ecb2..07083bf2d3e 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Percentile.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Percentile.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.sql.expression.function.aggregate; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.Expressions; import org.elasticsearch.xpack.sql.expression.Foldables; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.type.DataType; @@ -27,9 +28,9 @@ public class Percentile extends NumericAggregate implements EnclosedAgg { TypeResolution resolution = super.resolveType(); if (TypeResolution.TYPE_RESOLVED.equals(resolution)) { - resolution = percent().dataType().isNumeric() ? TypeResolution.TYPE_RESOLVED : - new TypeResolution("Percentile#percent argument cannot be non-numeric (type is'%s')", percent().dataType().esName()); + resolution = Expressions.typeMustBeNumeric(percent()); } + return resolution; } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/PercentileRank.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/PercentileRank.java index 8ed75e4ac52..ccdd8333a02 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/PercentileRank.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/PercentileRank.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.sql.expression.function.aggregate; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.Expressions; import org.elasticsearch.xpack.sql.expression.Foldables; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.type.DataType; @@ -27,9 +28,9 @@ public class PercentileRank extends AggregateFunction implements EnclosedAgg { TypeResolution resolution = super.resolveType(); if (TypeResolution.TYPE_RESOLVED.equals(resolution)) { - resolution = value.dataType().isNumeric() ? TypeResolution.TYPE_RESOLVED : - new TypeResolution("PercentileRank#value argument cannot be non-numeric (type is'%s')", value.dataType().esName()); + resolution = Expressions.typeMustBeNumeric(value); } + return resolution; } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/PercentileRanks.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/PercentileRanks.java index d87959985c6..651a517be17 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/PercentileRanks.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/PercentileRanks.java @@ -9,7 +9,6 @@ import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.tree.Location; import java.util.List; -import java.util.Objects; public class PercentileRanks extends CompoundNumericAggregate { @@ -23,19 +22,4 @@ public class PercentileRanks extends CompoundNumericAggregate { public List values() { return values; } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - - if (obj == null || getClass() != obj.getClass()) { - return false; - } - - PercentileRanks other = (PercentileRanks) obj; - return Objects.equals(field(), other.field()) - && Objects.equals(values, other.values); - } } \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Percentiles.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Percentiles.java index e49531e944c..876fe7fa1d6 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Percentiles.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Percentiles.java @@ -9,7 +9,6 @@ import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.tree.Location; import java.util.List; -import java.util.Objects; public class Percentiles extends CompoundNumericAggregate { @@ -23,19 +22,4 @@ public class Percentiles extends CompoundNumericAggregate { public List percents() { return percents; } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - - if (obj == null || getClass() != obj.getClass()) { - return false; - } - - Percentiles other = (Percentiles) obj; - return Objects.equals(field(), other.field()) - && Objects.equals(percents, other.percents); - } } \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/BinaryScalarFunction.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/BinaryScalarFunction.java new file mode 100644 index 00000000000..f5cd4d357e2 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/BinaryScalarFunction.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar; + +import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; +import org.elasticsearch.xpack.sql.expression.Attribute; +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.Expressions; +import org.elasticsearch.xpack.sql.expression.FieldAttribute; +import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunctionAttribute; +import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate; +import org.elasticsearch.xpack.sql.tree.Location; + +import java.util.Arrays; + +import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ParamsBuilder.paramsBuilder; +import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate.formatTemplate; + +public abstract class BinaryScalarFunction extends ScalarFunction { + + private final Expression left, right; + + protected BinaryScalarFunction(Location location, Expression left, Expression right) { + super(location, Arrays.asList(left, right)); + this.left = left; + this.right = right; + } + + public Expression left() { + return left; + } + + public Expression right() { + return right; + } + + public boolean foldable() { + return left.foldable() && right.foldable(); + } + + @Override + public ScalarFunctionAttribute toAttribute() { + return new ScalarFunctionAttribute(location(), name(), dataType(), id(), asScript(), orderBy(), asProcessor()); + } + + protected ScriptTemplate asScript() { + ScriptTemplate leftScript = asScript(left()); + ScriptTemplate rightScript = asScript(right()); + + return asScriptFrom(leftScript, rightScript); + } + + protected abstract ScriptTemplate asScriptFrom(ScriptTemplate leftScript, ScriptTemplate rightScript); + + protected ScriptTemplate asScript(Expression exp) { + if (exp.foldable()) { + return asScriptFromFoldable(exp); + } + + Attribute attr = Expressions.attribute(exp); + if (attr != null) { + if (attr instanceof ScalarFunctionAttribute) { + return asScriptFrom((ScalarFunctionAttribute) attr); + } + if (attr instanceof AggregateFunctionAttribute) { + return asScriptFrom((AggregateFunctionAttribute) attr); + } + // fall-back to + return asScriptFrom((FieldAttribute) attr); + } + throw new SqlIllegalArgumentException("Cannot evaluate script for field %s", exp); + } + + protected ScriptTemplate asScriptFrom(ScalarFunctionAttribute scalar) { + return scalar.script(); + } + + protected ScriptTemplate asScriptFrom(AggregateFunctionAttribute aggregate) { + return new ScriptTemplate(formatTemplate("{}"), + paramsBuilder().agg(aggregate.functionId(), aggregate.propertyPath()).build(), + aggregate.dataType()); + } + + protected ScriptTemplate asScriptFrom(FieldAttribute field) { + return new ScriptTemplate(formatTemplate("doc[{}].value"), + paramsBuilder().variable(field.name()).build(), + field.dataType()); + } + + protected ScriptTemplate asScriptFromFoldable(Expression foldable) { + return new ScriptTemplate(formatTemplate("{}"), + paramsBuilder().variable(foldable.fold()).build(), + foldable.dataType()); + } +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Cast.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Cast.java index cc9edebaf3e..22e6ce941f9 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Cast.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Cast.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.sql.expression.function.scalar; import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.expression.FieldAttribute; import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunctionAttribute; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.UnaryProcessorDefinition; import org.elasticsearch.xpack.sql.expression.function.scalar.script.Params; import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate; import org.elasticsearch.xpack.sql.tree.Location; @@ -19,17 +22,17 @@ import java.util.Objects; import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ParamsBuilder.paramsBuilder; import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate.formatTemplate; -public class Cast extends ScalarFunction { +public class Cast extends UnaryScalarFunction { private final DataType dataType; - public Cast(Location location, Expression argument, DataType dataType) { - super(location, argument); + public Cast(Location location, Expression field, DataType dataType) { + super(location, field); this.dataType = dataType; } public DataType from() { - return argument().dataType(); + return field().dataType(); } public DataType to() { @@ -41,9 +44,19 @@ public class Cast extends ScalarFunction { return dataType; } + @Override + public boolean foldable() { + return field().foldable(); + } + + @Override + public Object fold() { + return DataTypeConversion.convert(field().fold(), dataType); + } + @Override public boolean nullable() { - return argument().nullable() || DataTypeConversion.nullable(from(), to()); + return field().nullable() || DataTypeConversion.nullable(from()); } @Override @@ -77,8 +90,8 @@ public class Cast extends ScalarFunction { } @Override - public ColumnProcessor asProcessor() { - return new CastProcessor(DataTypeConversion.conversionFor(from(), to())); + protected ProcessorDefinition makeProcessor() { + return new UnaryProcessorDefinition(this, ProcessorDefinitions.toProcessorDefinition(field()), new CastProcessor(DataTypeConversion.conversionFor(from(), to()))); } @Override @@ -88,6 +101,6 @@ public class Cast extends ScalarFunction { @Override public String toString() { - return functionName() + "(" + argument().toString() + " AS " + to().sqlName() + ")#" + id(); + return functionName() + "(" + field().toString() + " AS " + to().sqlName() + ")#" + id(); } } \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/CastProcessor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/CastProcessor.java index 8ac6bb271f6..f5fe541fb46 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/CastProcessor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/CastProcessor.java @@ -7,35 +7,39 @@ package org.elasticsearch.xpack.sql.expression.function.scalar; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; import org.elasticsearch.xpack.sql.type.DataTypeConversion.Conversion; import java.io.IOException; +import java.util.Objects; + +public class CastProcessor implements Processor { + + public static final String NAME = "ca"; -public class CastProcessor implements ColumnProcessor { - public static final String NAME = "c"; private final Conversion conversion; - CastProcessor(Conversion conversion) { + public CastProcessor(Conversion conversion) { this.conversion = conversion; } - CastProcessor(StreamInput in) throws IOException { + public CastProcessor(StreamInput in) throws IOException { conversion = in.readEnum(Conversion.class); } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeEnum(conversion); - } - @Override public String getWriteableName() { return NAME; } @Override - public Object apply(Object r) { - return conversion.convert(r); + public void writeTo(StreamOutput out) throws IOException { + out.writeEnum(conversion); + } + + @Override + public Object process(Object input) { + return conversion.convert(input); } Conversion converter() { @@ -44,20 +48,25 @@ public class CastProcessor implements ColumnProcessor { @Override public boolean equals(Object obj) { - if (obj == null || obj.getClass() != getClass()) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { return false; } + CastProcessor other = (CastProcessor) obj; - return conversion.equals(other.conversion); + return Objects.equals(conversion, other.conversion); } @Override public int hashCode() { - return conversion.hashCode(); + return Objects.hash(conversion); } @Override public String toString() { - return conversion.toString(); + return conversion.name(); } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ColumnProcessor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ColumnProcessor.java deleted file mode 100644 index 625bb0b7b92..00000000000 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ColumnProcessor.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.sql.expression.function.scalar; - -import org.elasticsearch.common.io.stream.NamedWriteable; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; - -import java.util.ArrayList; -import java.util.List; - -public interface ColumnProcessor extends NamedWriteable { - /** - * All of the named writeables needed to deserialize the instances - * of {@linkplain ColumnProcessor}. - */ - static List getNamedWriteables() { - List entries = new ArrayList<>(); - entries.add(new NamedWriteableRegistry.Entry(ColumnProcessor.class, CastProcessor.NAME, CastProcessor::new)); - entries.add(new NamedWriteableRegistry.Entry(ColumnProcessor.class, ComposeProcessor.NAME, ComposeProcessor::new)); - entries.add(new NamedWriteableRegistry.Entry(ColumnProcessor.class, DateTimeProcessor.NAME, DateTimeProcessor::new)); - entries.add(new NamedWriteableRegistry.Entry(ColumnProcessor.class, - MathFunctionProcessor.NAME, MathFunctionProcessor::new)); - entries.add(new NamedWriteableRegistry.Entry(ColumnProcessor.class, - MatrixFieldProcessor.NAME, MatrixFieldProcessor::new)); - return entries; - } - - Object apply(Object r); -} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ComposeProcessor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ComposeProcessor.java deleted file mode 100644 index 570bd2e594b..00000000000 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ComposeProcessor.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.sql.expression.function.scalar; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; - -import java.io.IOException; -import java.util.Objects; - -/** - * A {@linkplain ColumnProcessor} that composes the results of two - * {@linkplain ColumnProcessor}s. - */ -public class ComposeProcessor implements ColumnProcessor { - static final String NAME = "."; - private final ColumnProcessor first; - private final ColumnProcessor second; - - public ComposeProcessor(ColumnProcessor first, ColumnProcessor second) { - this.first = first; - this.second = second; - } - - public ComposeProcessor(StreamInput in) throws IOException { - first = in.readNamedWriteable(ColumnProcessor.class); - second = in.readNamedWriteable(ColumnProcessor.class); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeNamedWriteable(first); - out.writeNamedWriteable(second); - } - - @Override - public String getWriteableName() { - return NAME; - } - - @Override - public Object apply(Object r) { - return second.apply(first.apply(r)); - } - - ColumnProcessor first() { - return first; - } - - ColumnProcessor second() { - return second; - } - - @Override - public boolean equals(Object obj) { - if (obj == null || obj.getClass() != getClass()) { - return false; - } - ComposeProcessor other = (ComposeProcessor) obj; - return first.equals(other.first) - && second.equals(other.second); - } - - @Override - public int hashCode() { - return Objects.hash(first, second); - } - - @Override - public String toString() { - // borrow Haskell's notation for function comosition - return "(" + second + " . " + first + ")"; - } -} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/MathFunctionProcessor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/MathFunctionProcessor.java deleted file mode 100644 index ad0b9044008..00000000000 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/MathFunctionProcessor.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.sql.expression.function.scalar; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor; - -import java.io.IOException; - -public class MathFunctionProcessor implements ColumnProcessor { - public static final String NAME = "m"; - - private final MathProcessor processor; - - public MathFunctionProcessor(MathProcessor processor) { - this.processor = processor; - } - - MathFunctionProcessor(StreamInput in) throws IOException { - processor = in.readEnum(MathProcessor.class); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeEnum(processor); - } - - @Override - public String getWriteableName() { - return NAME; - } - - @Override - public Object apply(Object r) { - return processor.apply(r); - } - - MathProcessor processor() { - return processor; - } - - @Override - public boolean equals(Object obj) { - if (obj == null || obj.getClass() != getClass()) { - return false; - } - MathFunctionProcessor other = (MathFunctionProcessor) obj; - return processor == other.processor; - } - - @Override - public int hashCode() { - return processor.hashCode(); - } - - @Override - public String toString() { - return processor.toString(); - } -} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Processors.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Processors.java new file mode 100644 index 00000000000..4534792ce4b --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Processors.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; +import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.BinaryArithmeticProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.UnaryArithmeticProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.ChainingProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.ConstantProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.HitExtractorProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.MatrixFieldProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; + +import java.util.ArrayList; +import java.util.List; + +public abstract class Processors { + + /** + * All of the named writeables needed to deserialize the instances of + * {@linkplain Processors}. + */ + public static List getNamedWriteables() { + List entries = new ArrayList<>(); + // base + entries.add(new Entry(Processor.class, ConstantProcessor.NAME, ConstantProcessor::new)); + entries.add(new Entry(Processor.class, HitExtractorProcessor.NAME, HitExtractorProcessor::new)); + entries.add(new Entry(Processor.class, CastProcessor.NAME, CastProcessor::new)); + entries.add(new Entry(Processor.class, ChainingProcessor.NAME, ChainingProcessor::new)); + entries.add(new Entry(Processor.class, MatrixFieldProcessor.NAME, MatrixFieldProcessor::new)); + + // arithmetic + entries.add(new Entry(Processor.class, BinaryArithmeticProcessor.NAME, BinaryArithmeticProcessor::new)); + entries.add(new Entry(Processor.class, UnaryArithmeticProcessor.NAME, UnaryArithmeticProcessor::new)); + // datetime + entries.add(new Entry(Processor.class, DateTimeProcessor.NAME, DateTimeProcessor::new)); + // math + entries.add(new Entry(Processor.class, MathProcessor.NAME, MathProcessor::new)); + return entries; + } +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ScalarFunction.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ScalarFunction.java index 861c73d376f..e0f444c923a 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ScalarFunction.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ScalarFunction.java @@ -5,83 +5,41 @@ */ package org.elasticsearch.xpack.sql.expression.function.scalar; -import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; -import org.elasticsearch.xpack.sql.expression.Attribute; import org.elasticsearch.xpack.sql.expression.Expression; -import org.elasticsearch.xpack.sql.expression.Expressions; -import org.elasticsearch.xpack.sql.expression.FieldAttribute; import org.elasticsearch.xpack.sql.expression.function.Function; -import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunctionAttribute; -import org.elasticsearch.xpack.sql.expression.function.scalar.script.Params; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition; import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate; import org.elasticsearch.xpack.sql.tree.Location; -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; +import java.util.List; -import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ParamsBuilder.paramsBuilder; +import static java.util.Collections.emptyList; public abstract class ScalarFunction extends Function { - private final Expression argument; + private ProcessorDefinition lazyProcessor = null; protected ScalarFunction(Location location) { super(location, emptyList()); - this.argument = null; } - protected ScalarFunction(Location location, Expression child) { - super(location, singletonList(child)); - this.argument = child; - } - - public Expression argument() { - return argument; + protected ScalarFunction(Location location, List fields) { + super(location, fields); } @Override - public ScalarFunctionAttribute toAttribute() { - String functionId = null; - Attribute attr = Expressions.attribute(argument()); + public abstract ScalarFunctionAttribute toAttribute(); - if (attr instanceof AggregateFunctionAttribute) { - AggregateFunctionAttribute afa = (AggregateFunctionAttribute) attr; - functionId = afa.functionId(); + protected abstract ScriptTemplate asScript(); + + public ProcessorDefinition asProcessor() { + if (lazyProcessor == null) { + lazyProcessor = makeProcessor(); } - - return new ScalarFunctionAttribute(location(), name(), dataType(), id(), asScript(), orderBy(), functionId); + return lazyProcessor; } - protected ScriptTemplate asScript() { - Attribute attr = Expressions.attribute(argument()); - if (attr != null) { - if (attr instanceof ScalarFunctionAttribute) { - return asScriptFrom((ScalarFunctionAttribute) attr); - } - if (attr instanceof AggregateFunctionAttribute) { - return asScriptFrom((AggregateFunctionAttribute) attr); - } - - // fall-back to - return asScriptFrom((FieldAttribute) attr); - } - throw new SqlIllegalArgumentException("Cannot evaluate script for field %s", argument()); - } - - protected ScriptTemplate asScriptFrom(ScalarFunctionAttribute scalar) { - ScriptTemplate nested = scalar.script(); - Params p = paramsBuilder().script(nested.params()).build(); - return new ScriptTemplate(chainScalarTemplate(nested.template()), p, dataType()); - } - - protected abstract ScriptTemplate asScriptFrom(AggregateFunctionAttribute aggregate); - - protected abstract ScriptTemplate asScriptFrom(FieldAttribute field); - - protected abstract String chainScalarTemplate(String template); - - - public abstract ColumnProcessor asProcessor(); + protected abstract ProcessorDefinition makeProcessor(); // used if the function is monotonic and thus does not have to be computed for ordering purposes public Expression orderBy() { diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ScalarFunctionAttribute.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ScalarFunctionAttribute.java index cf544054164..ea8c0558111 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ScalarFunctionAttribute.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ScalarFunctionAttribute.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.sql.expression.Attribute; import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.expression.ExpressionId; import org.elasticsearch.xpack.sql.expression.TypedAttribute; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition; import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.type.DataType; @@ -17,17 +18,17 @@ public class ScalarFunctionAttribute extends TypedAttribute { private final ScriptTemplate script; private final Expression orderBy; - private final String functionId; + private final ProcessorDefinition processorDef; - ScalarFunctionAttribute(Location location, String name, DataType dataType, ExpressionId id, ScriptTemplate script, Expression orderBy, String functionId) { - this(location, name, dataType, null, true, id, false, script, orderBy, functionId); + ScalarFunctionAttribute(Location location, String name, DataType dataType, ExpressionId id, ScriptTemplate script, Expression orderBy, ProcessorDefinition processorDef) { + this(location, name, dataType, null, true, id, false, script, orderBy, processorDef); } - ScalarFunctionAttribute(Location location, String name, DataType dataType, String qualifier, boolean nullable, ExpressionId id, boolean synthetic, ScriptTemplate script, Expression orderBy, String functionId) { + ScalarFunctionAttribute(Location location, String name, DataType dataType, String qualifier, boolean nullable, ExpressionId id, boolean synthetic, ScriptTemplate script, Expression orderBy, ProcessorDefinition processorDef) { super(location, name, dataType, qualifier, nullable, id, synthetic); this.script = script; this.orderBy = orderBy; - this.functionId = functionId; + this.processorDef = processorDef; } public ScriptTemplate script() { @@ -38,18 +39,18 @@ public class ScalarFunctionAttribute extends TypedAttribute { return orderBy; } - public String functionId() { - return functionId; + public ProcessorDefinition processorDef() { + return processorDef; } @Override protected Expression canonicalize() { - return new ScalarFunctionAttribute(location(), "", dataType(), null, true, id(), false, script, orderBy, functionId); + return new ScalarFunctionAttribute(location(), "", dataType(), null, true, id(), false, script, orderBy, processorDef); } @Override protected Attribute clone(Location location, String name, DataType dataType, String qualifier, boolean nullable, ExpressionId id, boolean synthetic) { - return new ScalarFunctionAttribute(location, name, dataType, qualifier, nullable, id, synthetic, script, orderBy, functionId); + return new ScalarFunctionAttribute(location, name, dataType, qualifier, nullable, id, synthetic, script, orderBy, processorDef); } @Override diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/UnaryScalarFunction.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/UnaryScalarFunction.java new file mode 100644 index 00000000000..94027e82d2f --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/UnaryScalarFunction.java @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar; + +import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; +import org.elasticsearch.xpack.sql.expression.Attribute; +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.Expressions; +import org.elasticsearch.xpack.sql.expression.FieldAttribute; +import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunctionAttribute; +import org.elasticsearch.xpack.sql.expression.function.scalar.script.Params; +import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate; +import org.elasticsearch.xpack.sql.tree.Location; + +import static java.util.Collections.singletonList; +import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ParamsBuilder.paramsBuilder; +import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate.formatTemplate; + +public abstract class UnaryScalarFunction extends ScalarFunction { + + private final Expression field; + + protected UnaryScalarFunction(Location location) { + super(location); + this.field = null; + } + + protected UnaryScalarFunction(Location location, Expression field) { + super(location, singletonList(field)); + this.field = field; + } + + public Expression field() { + return field; + } + + @Override + public boolean foldable() { + return field.foldable(); + } + + @Override + public ScalarFunctionAttribute toAttribute() { + String functionId = null; + Attribute attr = Expressions.attribute(field()); + + if (attr instanceof AggregateFunctionAttribute) { + AggregateFunctionAttribute afa = (AggregateFunctionAttribute) attr; + functionId = afa.functionId(); + } + + return new ScalarFunctionAttribute(location(), name(), dataType(), id(), asScript(), orderBy(), asProcessor()); + } + + protected ScriptTemplate asScript() { + if (field.foldable()) { + return asScriptFromFoldable(field); + } + + Attribute attr = Expressions.attribute(field()); + if (attr != null) { + if (attr instanceof ScalarFunctionAttribute) { + return asScriptFrom((ScalarFunctionAttribute) attr); + } + if (attr instanceof AggregateFunctionAttribute) { + return asScriptFrom((AggregateFunctionAttribute) attr); + } + + // fall-back to + return asScriptFrom((FieldAttribute) attr); + } + throw new SqlIllegalArgumentException("Cannot evaluate script for field %s", field()); + } + + protected ScriptTemplate asScriptFromFoldable(Expression foldable) { + return new ScriptTemplate(formatTemplate("{}"), + paramsBuilder().variable(foldable.fold()).build(), + foldable.dataType()); + } + + protected ScriptTemplate asScriptFrom(ScalarFunctionAttribute scalar) { + ScriptTemplate nested = scalar.script(); + Params p = paramsBuilder().script(nested.params()).build(); + return new ScriptTemplate(chainScalarTemplate(nested.template()), p, dataType()); + } + + protected abstract ScriptTemplate asScriptFrom(AggregateFunctionAttribute aggregate); + + protected abstract ScriptTemplate asScriptFrom(FieldAttribute field); + + protected abstract String chainScalarTemplate(String template); +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Add.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Add.java new file mode 100644 index 00000000000..7df251780b2 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Add.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.BinaryArithmeticProcessor.BinaryArithmeticOperation; +import org.elasticsearch.xpack.sql.tree.Location; + +public class Add extends ArithmeticFunction { + + public Add(Location location, Expression left, Expression right) { + super(location, left, right, BinaryArithmeticOperation.ADD); + } + + @Override + public Number fold() { + return Arithmetics.add((Number) left().fold(), (Number) right().fold()); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/ArithmeticFunction.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/ArithmeticFunction.java new file mode 100644 index 00000000000..ec402c825c5 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/ArithmeticFunction.java @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.Literal; +import org.elasticsearch.xpack.sql.expression.function.scalar.BinaryScalarFunction; +import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.BinaryArithmeticProcessor.BinaryArithmeticOperation; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions; +import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.type.DataType; + +import java.util.Locale; + +import static java.lang.String.format; +import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ParamsBuilder.paramsBuilder; + +public abstract class ArithmeticFunction extends BinaryScalarFunction { + + private BinaryArithmeticOperation operation; + + ArithmeticFunction(Location location, Expression left, Expression right, BinaryArithmeticOperation operation) { + super(location, left, right); + this.operation = operation; + } + + public BinaryArithmeticOperation operation() { + return operation; + } + + @Override + public DataType dataType() { + // left or right have to be compatible so either one works + return left().dataType(); + } + + @Override + protected TypeResolution resolveType() { + if (!childrenResolved()) { + return new TypeResolution("Unresolved children"); + } + DataType l = left().dataType(); + DataType r = right().dataType(); + + TypeResolution resolution = resolveInputType(l); + + if (resolution == TypeResolution.TYPE_RESOLVED) { + return resolveInputType(r); + } + return resolution; + } + + protected TypeResolution resolveInputType(DataType inputType) { + return inputType.isNumeric() ? TypeResolution.TYPE_RESOLVED + : new TypeResolution("'%s' requires a numeric type, not %s", operation, inputType.sqlName()); + } + + @Override + protected ScriptTemplate asScriptFrom(ScriptTemplate leftScript, ScriptTemplate rightScript) { + return new ScriptTemplate(format(Locale.ROOT, "(%s) %s (%s)", leftScript.template(), operation.symbol(), rightScript.template()), + paramsBuilder().script(leftScript.params()).script(rightScript.params()).build(), + dataType()); + } + + protected final BinaryArithmeticProcessorDefinition makeProcessor() { + return new BinaryArithmeticProcessorDefinition(this, ProcessorDefinitions.toProcessorDefinition(left()), ProcessorDefinitions.toProcessorDefinition(right()), operation); + } + + @Override + public String name() { + return toString(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(left()); + if (!(left() instanceof Literal)) { + sb.insert(0, "("); + sb.append(")"); + } + sb.append(" "); + sb.append(operation); + sb.append(" "); + int pos = sb.length(); + sb.append(right()); + if (!(right() instanceof Literal)) { + sb.insert(pos, "("); + sb.append(")"); + } + return sb.toString(); + } + + protected boolean useParanthesis() { + return !(left() instanceof Literal) || !(right() instanceof Literal); + } +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Arithmetics.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Arithmetics.java new file mode 100644 index 00000000000..2bae1935e4f --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Arithmetics.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic; + +/** + * Arithmetic operation using the type widening rules of the JLS 5.6.2 namely + * widen to double or float or long or int in this order. + */ +abstract class Arithmetics { + + static Number add(Number l, Number r) { + if (l instanceof Double || r instanceof Double) { + return Double.valueOf(l.doubleValue() + r.doubleValue()); + } + if (l instanceof Float || r instanceof Float) { + return Float.valueOf(l.floatValue() + r.floatValue()); + } + if (l instanceof Long || r instanceof Long) { + return Long.valueOf(Math.addExact(l.longValue(), r.longValue())); + } + + return Integer.valueOf(Math.addExact(l.intValue(), r.intValue())); + } + + static Number sub(Number l, Number r) { + if (l instanceof Double || r instanceof Double) { + return Double.valueOf(l.doubleValue() - r.doubleValue()); + } + if (l instanceof Float || r instanceof Float) { + return Float.valueOf(l.floatValue() - r.floatValue()); + } + if (l instanceof Long || r instanceof Long) { + return Long.valueOf(Math.subtractExact(l.longValue(), r.longValue())); + } + + return Integer.valueOf(Math.subtractExact(l.intValue(), r.intValue())); + } + + static Number mul(Number l, Number r) { + if (l instanceof Double || r instanceof Double) { + return Double.valueOf(l.doubleValue() * r.doubleValue()); + } + if (l instanceof Float || r instanceof Float) { + return Float.valueOf(l.floatValue() * r.floatValue()); + } + if (l instanceof Long || r instanceof Long) { + return Long.valueOf(Math.multiplyExact(l.longValue(), r.longValue())); + } + + return Integer.valueOf(Math.multiplyExact(l.intValue(), r.intValue())); + } + + static Number div(Number l, Number r) { + if (l instanceof Double || r instanceof Double) { + return l.doubleValue() / r.doubleValue(); + } + if (l instanceof Float || r instanceof Float) { + return l.floatValue() / r.floatValue(); + } + if (l instanceof Long || r instanceof Long) { + return l.longValue() / r.longValue(); + } + + return l.intValue() / r.intValue(); + } + + static Number mod(Number l, Number r) { + if (l instanceof Long || r instanceof Long) { + return Long.valueOf(Math.floorMod(l.longValue(), r.longValue())); + } + if (l instanceof Double || r instanceof Double) { + return Double.valueOf(l.doubleValue() % r.doubleValue()); + } + if (l instanceof Float || r instanceof Float) { + return Float.valueOf(l.floatValue() % r.floatValue()); + } + + return Math.floorMod(l.intValue(), r.intValue()); + } + + static Number negate(Number n) { + if (n instanceof Double) { + double d = n.doubleValue(); + if (d == Double.MIN_VALUE) { + throw new ArithmeticException("double overflow"); + } + return Double.valueOf(-n.doubleValue()); + } + if (n instanceof Float) { + float f = n.floatValue(); + if (f == Float.MIN_VALUE) { + throw new ArithmeticException("float overflow"); + } + return Float.valueOf(-n.floatValue()); + } + if (n instanceof Long) { + return Long.valueOf(Math.negateExact(n.longValue())); + } + + return Integer.valueOf(Math.negateExact(n.intValue())); + } +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/BinaryArithmeticProcessor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/BinaryArithmeticProcessor.java new file mode 100644 index 00000000000..bfb4cd811cb --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/BinaryArithmeticProcessor.java @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.BinaryProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; + +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; +import java.util.function.BiFunction; + +import static java.lang.String.format; + +public class BinaryArithmeticProcessor extends BinaryProcessor { + + public enum BinaryArithmeticOperation { + + ADD(Arithmetics::add, "+"), + SUB(Arithmetics::sub, "-"), + MUL(Arithmetics::mul, "*"), + DIV(Arithmetics::div, "/"), + MOD(Arithmetics::mod, "%"); + + private final BiFunction process; + private final String symbol; + + BinaryArithmeticOperation(BiFunction process, String symbol) { + this.process = process; + this.symbol = symbol; + } + + public String symbol() { + return symbol; + } + + public final Number apply(Number left, Number right) { + return process.apply(left, right); + } + + @Override + public String toString() { + return symbol; + } + } + + public static final String NAME = "ab"; + + private final BinaryArithmeticOperation operation; + + public BinaryArithmeticProcessor(Processor left, Processor right, BinaryArithmeticOperation operation) { + super(left, right); + this.operation = operation; + } + + public BinaryArithmeticProcessor(StreamInput in) throws IOException { + super(in); + operation = in.readEnum(BinaryArithmeticOperation.class); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + protected void doWrite(StreamOutput out) throws IOException { + out.writeEnum(operation); + } + + @Override + protected Object doProcess(Object left, Object right) { + return operation.apply((Number) left, (Number) right); + } + + @Override + public int hashCode() { + return operation.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + BinaryArithmeticProcessor other = (BinaryArithmeticProcessor) obj; + return Objects.equals(operation, other.operation) + && Objects.equals(left(), other.left()) + && Objects.equals(right(), other.right()); + } + + @Override + public String toString() { + return format(Locale.ROOT, "(%s %s %s)", left(), operation, right()); + } +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/BinaryArithmeticProcessorDefinition.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/BinaryArithmeticProcessorDefinition.java new file mode 100644 index 00000000000..e0d60fe6ef5 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/BinaryArithmeticProcessorDefinition.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.BinaryArithmeticProcessor.BinaryArithmeticOperation; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.BinaryProcessorDefinition; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition; + +import java.util.Objects; + +public class BinaryArithmeticProcessorDefinition extends BinaryProcessorDefinition { + + private final BinaryArithmeticOperation operation; + + public BinaryArithmeticProcessorDefinition(Expression expression, ProcessorDefinition left, ProcessorDefinition right, BinaryArithmeticOperation operation) { + super(expression, left, right); + this.operation = operation; + } + + public BinaryArithmeticOperation operation() { + return operation; + } + + @Override + public BinaryArithmeticProcessor asProcessor() { + return new BinaryArithmeticProcessor(left().asProcessor(), right().asProcessor(), operation); + } + + @Override + public int hashCode() { + return Objects.hash(left(), right(), operation); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + BinaryArithmeticProcessorDefinition other = (BinaryArithmeticProcessorDefinition) obj; + return Objects.equals(operation, other.operation) + && Objects.equals(left(), other.left()) + && Objects.equals(right(), other.right()); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Div.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Div.java new file mode 100644 index 00000000000..d4b084483a8 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Div.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.BinaryArithmeticProcessor.BinaryArithmeticOperation; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DataTypeConversion; + +public class Div extends ArithmeticFunction { + + public Div(Location location, Expression left, Expression right) { + super(location, left, right, BinaryArithmeticOperation.DIV); + } + + @Override + public Object fold() { + return Arithmetics.div((Number) left().fold(), (Number) right().fold()); + } + + @Override + public DataType dataType() { + return DataTypeConversion.commonType(left().dataType(), right().dataType()); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Mod.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Mod.java new file mode 100644 index 00000000000..59ac868bae2 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Mod.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.BinaryArithmeticProcessor.BinaryArithmeticOperation; +import org.elasticsearch.xpack.sql.tree.Location; + +public class Mod extends ArithmeticFunction { + + public Mod(Location location, Expression left, Expression right) { + super(location, left, right, BinaryArithmeticOperation.MOD); + } + + @Override + public Object fold() { + return Arithmetics.mod((Number) left().fold(), (Number) right().fold()); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Mul.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Mul.java new file mode 100644 index 00000000000..3120fe11384 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Mul.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.BinaryArithmeticProcessor.BinaryArithmeticOperation; +import org.elasticsearch.xpack.sql.tree.Location; + +public class Mul extends ArithmeticFunction { + + public Mul(Location location, Expression left, Expression right) { + super(location, left, right, BinaryArithmeticOperation.MUL); + } + + @Override + public Object fold() { + return Arithmetics.mul((Number) left().fold(), (Number) right().fold()); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Neg.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Neg.java new file mode 100644 index 00000000000..a35b36f0e3b --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Neg.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.Expressions; +import org.elasticsearch.xpack.sql.expression.FieldAttribute; +import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunctionAttribute; +import org.elasticsearch.xpack.sql.expression.function.scalar.UnaryScalarFunction; +import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.UnaryArithmeticProcessor.UnaryArithmeticOperation; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.UnaryProcessorDefinition; +import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.type.DataType; + +import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ParamsBuilder.paramsBuilder; +import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate.formatTemplate; + +public class Neg extends UnaryScalarFunction { + + public Neg(Location location, Expression field) { + super(location, field); + } + + @Override + protected TypeResolution resolveType() { + return Expressions.typeMustBeNumeric(field()); + } + + @Override + public Object fold() { + return Arithmetics.negate((Number) field().fold()); + } + + @Override + public DataType dataType() { + return field().dataType(); + } + + @Override + protected ScriptTemplate asScriptFrom(AggregateFunctionAttribute aggregate) { + return new ScriptTemplate(formatTemplate("{}"), + paramsBuilder().agg(aggregate.functionId(), aggregate.propertyPath()).build(), + dataType()); + } + + @Override + protected ScriptTemplate asScriptFrom(FieldAttribute field) { + return new ScriptTemplate(formatTemplate("doc[{}].value"), + paramsBuilder().variable(field.name()).build(), + dataType()); + } + + @Override + protected String chainScalarTemplate(String template) { + return template; + } + + @Override + protected ProcessorDefinition makeProcessor() { + return new UnaryProcessorDefinition(this, ProcessorDefinitions.toProcessorDefinition(field()), new UnaryArithmeticProcessor(UnaryArithmeticOperation.NEGATE)); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Sub.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Sub.java new file mode 100644 index 00000000000..64f08c4d452 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/Sub.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.BinaryArithmeticProcessor.BinaryArithmeticOperation; +import org.elasticsearch.xpack.sql.tree.Location; + +public class Sub extends ArithmeticFunction { + + public Sub(Location location, Expression left, Expression right) { + super(location, left, right, BinaryArithmeticOperation.SUB); + } + + @Override + public Object fold() { + return Arithmetics.sub((Number) left().fold(), (Number) right().fold()); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/UnaryArithmeticProcessor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/UnaryArithmeticProcessor.java new file mode 100644 index 00000000000..cc60a8c5004 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/UnaryArithmeticProcessor.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; + +import java.io.IOException; +import java.util.function.Function; + +public class UnaryArithmeticProcessor implements Processor { + + public enum UnaryArithmeticOperation { + + NEGATE(Arithmetics::negate); + + private final Function process; + + UnaryArithmeticOperation(Function process) { + this.process = process; + } + + public final Number apply(Number number) { + return process.apply(number); + } + + public String symbol() { + return "-"; + } + } + + public static final String NAME = "au"; + + private final UnaryArithmeticOperation operation; + + public UnaryArithmeticProcessor(UnaryArithmeticOperation operation) { + this.operation = operation; + } + + public UnaryArithmeticProcessor(StreamInput in) throws IOException { + operation = in.readEnum(UnaryArithmeticOperation.class); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeEnum(operation); + } + + @Override + public Object process(Object input) { + if (input instanceof Number) { + return operation.apply((Number) input); + } + throw new SqlIllegalArgumentException("A number is required; received %s", input); + } + + + @Override + public String toString() { + return operation.symbol() + super.toString(); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeExtractor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeExtractor.java deleted file mode 100644 index 9a3f179ce9d..00000000000 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeExtractor.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; - -import org.joda.time.DateTimeFieldType; -import org.joda.time.ReadableDateTime; - -/** - * Extracts portions of {@link ReadableDateTime}s. Note that the position in the enum is used for serialization. - */ -public enum DateTimeExtractor { - DAY_OF_MONTH(DateTimeFieldType.dayOfMonth()), - DAY_OF_WEEK(DateTimeFieldType.dayOfWeek()), - DAY_OF_YEAR(DateTimeFieldType.dayOfYear()), - HOUR_OF_DAY(DateTimeFieldType.hourOfDay()), - MINUTE_OF_DAY(DateTimeFieldType.minuteOfDay()), - MINUTE_OF_HOUR(DateTimeFieldType.minuteOfHour()), - MONTH_OF_YEAR(DateTimeFieldType.monthOfYear()), - SECOND_OF_MINUTE(DateTimeFieldType.secondOfMinute()), - WEEK_OF_YEAR(DateTimeFieldType.weekOfWeekyear()), - YEAR(DateTimeFieldType.year()); - - private final DateTimeFieldType field; - - DateTimeExtractor(DateTimeFieldType field) { - this.field = field; - } - - public int extract(ReadableDateTime dt) { - return dt.get(field); - } -} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFunction.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFunction.java index 249c0a30cc0..8f1ea161eee 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFunction.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFunction.java @@ -10,9 +10,11 @@ import org.elasticsearch.xpack.sql.expression.Expressions; import org.elasticsearch.xpack.sql.expression.FieldAttribute; import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunctionAttribute; import org.elasticsearch.xpack.sql.expression.function.aware.TimeZoneAware; -import org.elasticsearch.xpack.sql.expression.function.scalar.ColumnProcessor; -import org.elasticsearch.xpack.sql.expression.function.scalar.DateTimeProcessor; -import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction; +import org.elasticsearch.xpack.sql.expression.function.scalar.UnaryScalarFunction; +import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeProcessor.DateTimeExtractor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.UnaryProcessorDefinition; import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.type.DataType; @@ -26,12 +28,12 @@ import static java.lang.String.format; import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ParamsBuilder.paramsBuilder; import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate.formatTemplate; -public abstract class DateTimeFunction extends ScalarFunction implements TimeZoneAware { +public abstract class DateTimeFunction extends UnaryScalarFunction implements TimeZoneAware { private final DateTimeZone timeZone; - public DateTimeFunction(Location location, Expression argument, DateTimeZone timeZone) { - super(location, argument); + public DateTimeFunction(Location location, Expression field, DateTimeZone timeZone) { + super(location, field); this.timeZone = timeZone; } @@ -39,11 +41,15 @@ public abstract class DateTimeFunction extends ScalarFunction implements TimeZon return timeZone; } + public boolean foldable() { + return field().foldable(); + } + @Override protected TypeResolution resolveType() { - return argument().dataType().same(DataTypes.DATE) ? + return field().dataType().same(DataTypes.DATE) ? TypeResolution.TYPE_RESOLVED : - new TypeResolution("Function '%s' cannot be applied on a non-date expression ('%s' of type '%s')", functionName(), Expressions.name(argument()), argument().dataType().esName()); + new TypeResolution("Function '%s' cannot be applied on a non-date expression ('%s' of type '%s')", functionName(), Expressions.name(field()), field().dataType().esName()); } @Override @@ -84,9 +90,8 @@ public abstract class DateTimeFunction extends ScalarFunction implements TimeZon return getClass().getSimpleName(); } - @Override - public final ColumnProcessor asProcessor() { - return new DateTimeProcessor(extractor(), timeZone); + protected final ProcessorDefinition makeProcessor() { + return new UnaryProcessorDefinition(this, ProcessorDefinitions.toProcessorDefinition(field()), new DateTimeProcessor(extractor(), timeZone)); } protected abstract DateTimeExtractor extractor(); diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/DateTimeProcessor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeProcessor.java similarity index 55% rename from sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/DateTimeProcessor.java rename to sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeProcessor.java index 762c8207a88..461bd0049e3 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/DateTimeProcessor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeProcessor.java @@ -3,19 +3,45 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.sql.expression.function.scalar; +package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeExtractor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; import org.joda.time.DateTime; +import org.joda.time.DateTimeFieldType; import org.joda.time.DateTimeZone; import org.joda.time.ReadableDateTime; import java.io.IOException; +import java.util.Objects; -public class DateTimeProcessor implements ColumnProcessor { - public static final String NAME = "d"; +public class DateTimeProcessor implements Processor { + + public enum DateTimeExtractor { + DAY_OF_MONTH(DateTimeFieldType.dayOfMonth()), + DAY_OF_WEEK(DateTimeFieldType.dayOfWeek()), + DAY_OF_YEAR(DateTimeFieldType.dayOfYear()), + HOUR_OF_DAY(DateTimeFieldType.hourOfDay()), + MINUTE_OF_DAY(DateTimeFieldType.minuteOfDay()), + MINUTE_OF_HOUR(DateTimeFieldType.minuteOfHour()), + MONTH_OF_YEAR(DateTimeFieldType.monthOfYear()), + SECOND_OF_MINUTE(DateTimeFieldType.secondOfMinute()), + WEEK_OF_YEAR(DateTimeFieldType.weekOfWeekyear()), + YEAR(DateTimeFieldType.year()); + + private final DateTimeFieldType field; + + DateTimeExtractor(DateTimeFieldType field) { + this.field = field; + } + + public int extract(ReadableDateTime dt) { + return dt.get(field); + } + } + + public static final String NAME = "dt"; private final DateTimeExtractor extractor; private final DateTimeZone timeZone; @@ -25,7 +51,7 @@ public class DateTimeProcessor implements ColumnProcessor { this.timeZone = timeZone; } - DateTimeProcessor(StreamInput in) throws IOException { + public DateTimeProcessor(StreamInput in) throws IOException { extractor = in.readEnum(DateTimeExtractor.class); timeZone = DateTimeZone.forID(in.readString()); } @@ -46,7 +72,11 @@ public class DateTimeProcessor implements ColumnProcessor { } @Override - public Object apply(Object l) { + public Object process(Object l) { + if (l == null) { + return null; + } + ReadableDateTime dt; // most dates are returned as long if (l instanceof Long) { @@ -55,28 +85,29 @@ public class DateTimeProcessor implements ColumnProcessor { else { dt = (ReadableDateTime) l; } - if (!timeZone.getID().equals("UTC")) { + if (!DateTimeZone.UTC.equals(timeZone)) { dt = dt.toDateTime().withZone(timeZone); } return extractor.extract(dt); } + @Override + public int hashCode() { + return Objects.hash(extractor, timeZone); + } + @Override public boolean equals(Object obj) { if (obj == null || obj.getClass() != getClass()) { return false; } DateTimeProcessor other = (DateTimeProcessor) obj; - return extractor == other.extractor; - } - - @Override - public int hashCode() { - return extractor.hashCode(); + return Objects.equals(extractor, other.extractor) + && Objects.equals(timeZone, other.timeZone); } @Override public String toString() { return extractor.toString(); } -} +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DayOfMonth.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DayOfMonth.java index d56bec81be0..935fd58670a 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DayOfMonth.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DayOfMonth.java @@ -6,14 +6,15 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeProcessor.DateTimeExtractor; import org.elasticsearch.xpack.sql.tree.Location; import org.joda.time.DateTimeZone; import java.time.temporal.ChronoField; public class DayOfMonth extends DateTimeFunction { - public DayOfMonth(Location location, Expression argument, DateTimeZone timeZone) { - super(location, argument, timeZone); + public DayOfMonth(Location location, Expression field, DateTimeZone timeZone) { + super(location, field, timeZone); } @Override diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DayOfWeek.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DayOfWeek.java index 2ba1e0e228a..8e3b33a2867 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DayOfWeek.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DayOfWeek.java @@ -6,14 +6,15 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeProcessor.DateTimeExtractor; import org.elasticsearch.xpack.sql.tree.Location; import org.joda.time.DateTimeZone; import java.time.temporal.ChronoField; public class DayOfWeek extends DateTimeFunction { - public DayOfWeek(Location location, Expression argument, DateTimeZone timeZone) { - super(location, argument, timeZone); + public DayOfWeek(Location location, Expression field, DateTimeZone timeZone) { + super(location, field, timeZone); } @Override diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DayOfYear.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DayOfYear.java index 2df87381fea..a3d4d1fc04d 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DayOfYear.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DayOfYear.java @@ -6,14 +6,15 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeProcessor.DateTimeExtractor; import org.elasticsearch.xpack.sql.tree.Location; import org.joda.time.DateTimeZone; import java.time.temporal.ChronoField; public class DayOfYear extends DateTimeFunction { - public DayOfYear(Location location, Expression argument, DateTimeZone timeZone) { - super(location, argument, timeZone); + public DayOfYear(Location location, Expression field, DateTimeZone timeZone) { + super(location, field, timeZone); } @Override diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/HourOfDay.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/HourOfDay.java index 673525012e1..45e1a44f0f5 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/HourOfDay.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/HourOfDay.java @@ -6,14 +6,15 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeProcessor.DateTimeExtractor; import org.elasticsearch.xpack.sql.tree.Location; import org.joda.time.DateTimeZone; import java.time.temporal.ChronoField; public class HourOfDay extends DateTimeFunction { - public HourOfDay(Location location, Expression argument, DateTimeZone timeZone) { - super(location, argument, timeZone); + public HourOfDay(Location location, Expression field, DateTimeZone timeZone) { + super(location, field, timeZone); } @Override diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/MinuteOfDay.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/MinuteOfDay.java index 6a9b2765eda..3b23ab2c3ca 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/MinuteOfDay.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/MinuteOfDay.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeProcessor.DateTimeExtractor; import org.elasticsearch.xpack.sql.tree.Location; import org.joda.time.DateTimeZone; @@ -13,8 +14,8 @@ import java.time.temporal.ChronoField; public class MinuteOfDay extends DateTimeFunction { - public MinuteOfDay(Location location, Expression argument, DateTimeZone timeZone) { - super(location, argument, timeZone); + public MinuteOfDay(Location location, Expression field, DateTimeZone timeZone) { + super(location, field, timeZone); } @Override diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/MinuteOfHour.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/MinuteOfHour.java index 3fbdaa5f123..81204122d03 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/MinuteOfHour.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/MinuteOfHour.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeProcessor.DateTimeExtractor; import org.elasticsearch.xpack.sql.tree.Location; import org.joda.time.DateTimeZone; diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/MonthOfYear.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/MonthOfYear.java index 5fc43ecc177..41e65f4224b 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/MonthOfYear.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/MonthOfYear.java @@ -6,14 +6,15 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeProcessor.DateTimeExtractor; import org.elasticsearch.xpack.sql.tree.Location; import org.joda.time.DateTimeZone; import java.time.temporal.ChronoField; public class MonthOfYear extends DateTimeFunction { - public MonthOfYear(Location location, Expression argument, DateTimeZone timeZone) { - super(location, argument, timeZone); + public MonthOfYear(Location location, Expression field, DateTimeZone timeZone) { + super(location, field, timeZone); } @Override diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/SecondOfMinute.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/SecondOfMinute.java index 37ff9f95cb7..f7e6776946f 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/SecondOfMinute.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/SecondOfMinute.java @@ -6,14 +6,15 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeProcessor.DateTimeExtractor; import org.elasticsearch.xpack.sql.tree.Location; import org.joda.time.DateTimeZone; import java.time.temporal.ChronoField; public class SecondOfMinute extends DateTimeFunction { - public SecondOfMinute(Location location, Expression argument, DateTimeZone timeZone) { - super(location, argument, timeZone); + public SecondOfMinute(Location location, Expression field, DateTimeZone timeZone) { + super(location, field, timeZone); } @Override diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/WeekOfWeekYear.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/WeekOfWeekYear.java index d764b34ba60..a46452a5061 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/WeekOfWeekYear.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/WeekOfWeekYear.java @@ -6,14 +6,15 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeProcessor.DateTimeExtractor; import org.elasticsearch.xpack.sql.tree.Location; import org.joda.time.DateTimeZone; import java.time.temporal.ChronoField; public class WeekOfWeekYear extends DateTimeFunction { - public WeekOfWeekYear(Location location, Expression argument, DateTimeZone timeZone) { - super(location, argument, timeZone); + public WeekOfWeekYear(Location location, Expression field, DateTimeZone timeZone) { + super(location, field, timeZone); } @Override diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/Year.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/Year.java index 05e66abc10d..e58490dac3d 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/Year.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/Year.java @@ -6,14 +6,15 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeProcessor.DateTimeExtractor; import org.elasticsearch.xpack.sql.tree.Location; import org.joda.time.DateTimeZone; import java.time.temporal.ChronoField; public class Year extends DateTimeFunction { - public Year(Location location, Expression argument, DateTimeZone timeZone) { - super(location, argument, timeZone); + public Year(Location location, Expression field, DateTimeZone timeZone) { + super(location, field, timeZone); } @Override @@ -28,7 +29,7 @@ public class Year extends DateTimeFunction { @Override public Expression orderBy() { - return argument(); + return field(); } @Override @@ -39,5 +40,5 @@ public class Year extends DateTimeFunction { @Override protected DateTimeExtractor extractor() { return DateTimeExtractor.YEAR; - } +} } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/ACos.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/ACos.java index 9e6d8604675..a5c972d23d5 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/ACos.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/ACos.java @@ -6,15 +6,16 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; public class ACos extends MathFunction { - public ACos(Location location, Expression argument) { - super(location, argument); + public ACos(Location location, Expression field) { + super(location, field); } @Override - protected MathProcessor processor() { - return MathProcessor.ACOS; + protected MathOperation operation() { + return MathOperation.ACOS; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/ASin.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/ASin.java index 868d91c2d30..a082f17ffb2 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/ASin.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/ASin.java @@ -6,15 +6,16 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; public class ASin extends MathFunction { - public ASin(Location location, Expression argument) { - super(location, argument); + public ASin(Location location, Expression field) { + super(location, field); } @Override - protected MathProcessor processor() { - return MathProcessor.ASIN; + protected MathOperation operation() { + return MathOperation.ASIN; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/ATan.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/ATan.java index db728a2f354..3b8c8253b2a 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/ATan.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/ATan.java @@ -6,15 +6,16 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; public class ATan extends MathFunction { - public ATan(Location location, Expression argument) { - super(location, argument); + public ATan(Location location, Expression field) { + super(location, field); } @Override - protected MathProcessor processor() { - return MathProcessor.ATAN; + protected MathOperation operation() { + return MathOperation.ATAN; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Abs.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Abs.java index 6dc21fee3b6..60ae94e7f64 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Abs.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Abs.java @@ -6,21 +6,22 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.type.DataType; public class Abs extends MathFunction { - public Abs(Location location, Expression argument) { - super(location, argument); + public Abs(Location location, Expression field) { + super(location, field); } @Override - protected MathProcessor processor() { - return MathProcessor.ABS; + protected MathOperation operation() { + return MathOperation.ABS; } @Override public DataType dataType() { - return argument().dataType(); + return field().dataType(); } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Cbrt.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Cbrt.java index fc613706c08..e3a8f920323 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Cbrt.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Cbrt.java @@ -6,15 +6,16 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; public class Cbrt extends MathFunction { - public Cbrt(Location location, Expression argument) { - super(location, argument); + public Cbrt(Location location, Expression field) { + super(location, field); } @Override - protected MathProcessor processor() { - return MathProcessor.CBRT; + protected MathOperation operation() { + return MathOperation.CBRT; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Ceil.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Ceil.java index f7b2b05cbb9..e12ff38431f 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Ceil.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Ceil.java @@ -6,15 +6,23 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DataTypeConversion; public class Ceil extends MathFunction { - public Ceil(Location location, Expression argument) { - super(location, argument); + public Ceil(Location location, Expression field) { + super(location, field); } @Override - protected MathProcessor processor() { - return MathProcessor.CEIL; + protected MathOperation operation() { + return MathOperation.CEIL; + } + + @Override + public DataType dataType() { + return DataTypeConversion.asInteger(field().dataType()); } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Cos.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Cos.java index 90307ffd578..086f389797c 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Cos.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Cos.java @@ -6,15 +6,16 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; public class Cos extends MathFunction { - public Cos(Location location, Expression argument) { - super(location, argument); + public Cos(Location location, Expression field) { + super(location, field); } @Override - protected MathProcessor processor() { - return MathProcessor.COS; + protected MathOperation operation() { + return MathOperation.COS; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Cosh.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Cosh.java index 13dafada168..5c957df9fb7 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Cosh.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Cosh.java @@ -6,15 +6,16 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; public class Cosh extends MathFunction { - public Cosh(Location location, Expression argument) { - super(location, argument); + public Cosh(Location location, Expression field) { + super(location, field); } @Override - protected MathProcessor processor() { - return MathProcessor.COSH; + protected MathOperation operation() { + return MathOperation.COSH; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Degrees.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Degrees.java index fd911e6a452..f4feb49dcd7 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Degrees.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Degrees.java @@ -6,11 +6,12 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; public class Degrees extends MathFunction { - public Degrees(Location location, Expression argument) { - super(location, argument); + public Degrees(Location location, Expression field) { + super(location, field); } @Override @@ -19,7 +20,7 @@ public class Degrees extends MathFunction { } @Override - protected MathProcessor processor() { - return MathProcessor.DEGREES; + protected MathOperation operation() { + return MathOperation.DEGREES; } } \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/E.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/E.java index ef750caa818..458650081d9 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/E.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/E.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.util.StringUtils; @@ -30,7 +31,7 @@ public class E extends MathFunction { } @Override - protected MathProcessor processor() { - return MathProcessor.E; + protected MathOperation operation() { + return MathOperation.E; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Exp.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Exp.java index 0f1baaa6fe5..a9dacc62436 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Exp.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Exp.java @@ -6,15 +6,16 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; public class Exp extends MathFunction { - public Exp(Location location, Expression argument) { - super(location, argument); + public Exp(Location location, Expression field) { + super(location, field); } @Override - protected MathProcessor processor() { - return MathProcessor.EXP; + protected MathOperation operation() { + return MathOperation.EXP; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Expm1.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Expm1.java index 8f8e799ce7c..dce2797c3d1 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Expm1.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Expm1.java @@ -6,15 +6,16 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; public class Expm1 extends MathFunction { - public Expm1(Location location, Expression argument) { - super(location, argument); + public Expm1(Location location, Expression field) { + super(location, field); } @Override - protected MathProcessor processor() { - return MathProcessor.EXPM1; + protected MathOperation operation() { + return MathOperation.EXPM1; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Floor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Floor.java index b80d5838d05..eb0c52e16e1 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Floor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Floor.java @@ -6,15 +6,23 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DataTypeConversion; public class Floor extends MathFunction { - public Floor(Location location, Expression argument) { - super(location, argument); + public Floor(Location location, Expression field) { + super(location, field); } @Override - protected MathProcessor processor() { - return MathProcessor.FLOOR; + protected MathOperation operation() { + return MathOperation.FLOOR; + } + + @Override + public DataType dataType() { + return DataTypeConversion.asInteger(field().dataType()); } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Log.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Log.java index 9b5969e3a01..970d94ad169 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Log.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Log.java @@ -6,15 +6,16 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; public class Log extends MathFunction { - public Log(Location location, Expression argument) { - super(location, argument); + public Log(Location location, Expression field) { + super(location, field); } @Override - protected MathProcessor processor() { - return MathProcessor.LOG; + protected MathOperation operation() { + return MathOperation.LOG; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Log10.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Log10.java index ead21579704..3656a1ab651 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Log10.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Log10.java @@ -6,15 +6,16 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; public class Log10 extends MathFunction { - public Log10(Location location, Expression argument) { - super(location, argument); + public Log10(Location location, Expression field) { + super(location, field); } @Override - protected MathProcessor processor() { - return MathProcessor.LOG10; + protected MathOperation operation() { + return MathOperation.LOG10; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathFunction.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathFunction.java index 636535e77e3..06ab69b5027 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathFunction.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathFunction.java @@ -8,10 +8,12 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.expression.FieldAttribute; import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunctionAttribute; -import org.elasticsearch.xpack.sql.expression.function.scalar.ColumnProcessor; -import org.elasticsearch.xpack.sql.expression.function.scalar.MathFunctionProcessor; -import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunctionAttribute; +import org.elasticsearch.xpack.sql.expression.function.scalar.UnaryScalarFunction; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.UnaryProcessorDefinition; import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.type.DataType; @@ -23,27 +25,27 @@ import static java.lang.String.format; import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ParamsBuilder.paramsBuilder; import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate.formatTemplate; -public abstract class MathFunction extends ScalarFunction { +public abstract class MathFunction extends UnaryScalarFunction { protected MathFunction(Location location) { super(location); } - protected MathFunction(Location location, Expression argument) { - super(location, argument); + protected MathFunction(Location location, Expression field) { + super(location, field); } public boolean foldable() { - return argument().foldable(); + return field().foldable(); } - @Override protected String chainScalarTemplate(String template) { return createTemplate(template); } @Override + // TODO: isn't chain Scalar Template enough? protected ScriptTemplate asScriptFrom(ScalarFunctionAttribute scalar) { ScriptTemplate nested = scalar.script(); return new ScriptTemplate(createTemplate(nested.template()), @@ -79,9 +81,9 @@ public abstract class MathFunction extends ScalarFunction { } @Override - public final ColumnProcessor asProcessor() { - return new MathFunctionProcessor(processor()); + protected final ProcessorDefinition makeProcessor() { + return new UnaryProcessorDefinition(this, ProcessorDefinitions.toProcessorDefinition(field()), new MathProcessor(operation())); } - protected abstract MathProcessor processor(); + protected abstract MathOperation operation(); } \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathProcessor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathProcessor.java index a65867e3ec6..1548de6704a 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathProcessor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathProcessor.java @@ -5,56 +5,112 @@ */ package org.elasticsearch.xpack.sql.expression.function.scalar.math; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; + +import java.io.IOException; import java.util.function.DoubleFunction; import java.util.function.Function; -/** - * Applies a math function. Note that the order of the enum constants is used for serialization. - */ -public enum MathProcessor { - ABS((Object l) -> { - if (l instanceof Float) { - return Math.abs(((Float) l).floatValue()); +public class MathProcessor implements Processor { + + public enum MathOperation { + ABS((Object l) -> { + if (l instanceof Float) { + return Math.abs(((Float) l).floatValue()); + } + if (l instanceof Double) { + return Math.abs(((Double) l).doubleValue()); + } + long lo = ((Number) l).longValue(); + return lo >= 0 ? lo : lo == Long.MIN_VALUE ? Long.MAX_VALUE : -lo; + }), + + ACOS(Math::acos), + ASIN(Math::asin), + ATAN(Math::atan), + CBRT(Math::cbrt), + CEIL(Math::ceil), + COS(Math::cos), + COSH(Math::cosh), + DEGREES(Math::toDegrees), + E((Object l) -> Math.E), + EXP(Math::exp), + EXPM1(Math::expm1), + FLOOR(Math::floor), + LOG(Math::log), + LOG10(Math::log10), + PI((Object l) -> Math.PI), + RADIANS(Math::toRadians), + ROUND((DoubleFunction) Math::round), + SIN(Math::sin), + SINH(Math::sinh), + SQRT(Math::sqrt), + TAN(Math::tan); + + private final Function apply; + + MathOperation(Function apply) { + this.apply = apply; } - if (l instanceof Double) { - return Math.abs(((Double) l).doubleValue()); + + MathOperation(DoubleFunction apply) { + this.apply = (Object l) -> apply.apply(((Number) l).doubleValue()); } - long lo = ((Number) l).longValue(); - return lo >= 0 ? lo : lo == Long.MIN_VALUE ? Long.MAX_VALUE : -lo; - }), - ACOS(fromDouble(Math::acos)), - ASIN(fromDouble(Math::asin)), - ATAN(fromDouble(Math::atan)), - CBRT(fromDouble(Math::cbrt)), - CEIL(fromDouble(Math::ceil)), - COS(fromDouble(Math::cos)), - COSH(fromDouble(Math::cosh)), - DEGREES(fromDouble(Math::toDegrees)), - E((Object l) -> Math.E), - EXP(fromDouble(Math::exp)), - EXPM1(fromDouble(Math::expm1)), - FLOOR(fromDouble(Math::floor)), - LOG(fromDouble(Math::log)), - LOG10(fromDouble(Math::log10)), - PI((Object l) -> Math.PI), - RADIANS(fromDouble(Math::toRadians)), - ROUND(fromDouble(Math::round)), - SIN(fromDouble(Math::sin)), - SINH(fromDouble(Math::sinh)), - SQRT(fromDouble(Math::sqrt)), - TAN(fromDouble(Math::tan)); - private final Function apply; + public final Object apply(Object l) { + return apply.apply(l); + } + } + + public static final String NAME = "m"; - MathProcessor(Function apply) { - this.apply = apply; + private final MathOperation processor; + + public MathProcessor(MathOperation processor) { + this.processor = processor; } - private static Function fromDouble(DoubleFunction apply) { - return (Object l) -> apply.apply(((Number) l).doubleValue()); + public MathProcessor(StreamInput in) throws IOException { + processor = in.readEnum(MathOperation.class); } - public final Object apply(Object l) { - return apply.apply(l); + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeEnum(processor); } -} + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public Object process(Object input) { + return processor.apply(input); + } + + MathOperation processor() { + return processor; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + MathProcessor other = (MathProcessor) obj; + return processor == other.processor; + } + + @Override + public int hashCode() { + return processor.hashCode(); + } + + @Override + public String toString() { + return processor.toString(); + } +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Pi.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Pi.java index 8733d5631af..9d1c4a0e48f 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Pi.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Pi.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.util.StringUtils; @@ -30,7 +31,7 @@ public class Pi extends MathFunction { } @Override - protected MathProcessor processor() { - return MathProcessor.PI; + protected MathOperation operation() { + return MathOperation.PI; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Radians.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Radians.java index cf4c3509e01..a94efc916c4 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Radians.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Radians.java @@ -6,11 +6,12 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; public class Radians extends MathFunction { - public Radians(Location location, Expression argument) { - super(location, argument); + public Radians(Location location, Expression field) { + super(location, field); } @Override @@ -19,7 +20,7 @@ public class Radians extends MathFunction { } @Override - protected MathProcessor processor() { - return MathProcessor.RADIANS; + protected MathOperation operation() { + return MathOperation.RADIANS; } } \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Round.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Round.java index a3b2f36e830..43d2e20bec0 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Round.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Round.java @@ -6,22 +6,23 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.type.DataType; -import org.elasticsearch.xpack.sql.type.DataTypes; +import org.elasticsearch.xpack.sql.type.DataTypeConversion; public class Round extends MathFunction { - public Round(Location location, Expression argument) { - super(location, argument); + public Round(Location location, Expression field) { + super(location, field); + } + + @Override + protected MathOperation operation() { + return MathOperation.ROUND; } @Override public DataType dataType() { - return DataTypes.LONG; - } - - @Override - protected MathProcessor processor() { - return MathProcessor.ROUND; + return DataTypeConversion.asInteger(field().dataType()); } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Sin.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Sin.java index 1f66389c9ab..ccae5f6b7bf 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Sin.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Sin.java @@ -6,15 +6,16 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; public class Sin extends MathFunction { - public Sin(Location location, Expression argument) { - super(location, argument); + public Sin(Location location, Expression field) { + super(location, field); } @Override - protected MathProcessor processor() { - return MathProcessor.SIN; + protected MathOperation operation() { + return MathOperation.SIN; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Sinh.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Sinh.java index 3bf9c9397d6..07994951a77 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Sinh.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Sinh.java @@ -6,15 +6,16 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; public class Sinh extends MathFunction { - public Sinh(Location location, Expression argument) { - super(location, argument); + public Sinh(Location location, Expression field) { + super(location, field); } @Override - protected MathProcessor processor() { - return MathProcessor.SINH; + protected MathOperation operation() { + return MathOperation.SINH; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Sqrt.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Sqrt.java index 70b1ec49bfe..2fc79b2718f 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Sqrt.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Sqrt.java @@ -6,15 +6,16 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; public class Sqrt extends MathFunction { - public Sqrt(Location location, Expression argument) { - super(location, argument); + public Sqrt(Location location, Expression field) { + super(location, field); } @Override - protected MathProcessor processor() { - return MathProcessor.SQRT; + protected MathOperation operation() { + return MathOperation.SQRT; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Tan.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Tan.java index d53d1b4cdd3..5680052c41a 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Tan.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Tan.java @@ -6,15 +6,16 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.math; import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.tree.Location; public class Tan extends MathFunction { - public Tan(Location location, Expression argument) { - super(location, argument); + public Tan(Location location, Expression field) { + super(location, field); } @Override - protected MathProcessor processor() { - return MathProcessor.TAN; + protected MathOperation operation() { + return MathOperation.TAN; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/AggPathInput.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/AggPathInput.java new file mode 100644 index 00000000000..6d3f7eca261 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/AggPathInput.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition; + +import org.elasticsearch.xpack.sql.expression.Expression; + +import java.util.Objects; + +public class AggPathInput extends UnresolvedInput { + + private final String innerKey; + + public AggPathInput(Expression expression, String context) { + this(expression, context, null); + } + + public AggPathInput(Expression expression, String context, String innerKey) { + super(expression, context); + this.innerKey = innerKey; + } + + public String innerKey() { + return innerKey; + } + + @Override + public int hashCode() { + return Objects.hash(context(), innerKey); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + AggPathInput other = (AggPathInput) obj; + return Objects.equals(context(), other.context()) + && Objects.equals(innerKey, other.innerKey); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/AggValueInput.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/AggValueInput.java new file mode 100644 index 00000000000..e151668a37a --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/AggValueInput.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.MatrixFieldProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.SuppliedProcessor; + +import java.util.Objects; +import java.util.function.Supplier; + +public class AggValueInput extends LeafInput> { + + private final String innerKey; + private final Processor matrixProcessor; + + public AggValueInput(Expression expression, Supplier context, String innerKey) { + super(expression, context); + this.innerKey = innerKey; + this.matrixProcessor = innerKey != null ? new MatrixFieldProcessor(innerKey) : null; + } + + public String innerKey() { + return innerKey; + } + + @Override + public Processor asProcessor() { + return new SuppliedProcessor(() -> matrixProcessor != null ? matrixProcessor.process(context().get()) : context().get()); + } + + @Override + public int hashCode() { + return Objects.hash(context(), innerKey); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + AggValueInput other = (AggValueInput) obj; + return Objects.equals(context(), other.context()) + && Objects.equals(innerKey, other.innerKey); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/AttributeInput.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/AttributeInput.java new file mode 100644 index 00000000000..d495cbe720e --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/AttributeInput.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition; + +import org.elasticsearch.xpack.sql.expression.Attribute; +import org.elasticsearch.xpack.sql.expression.Expression; + +public class AttributeInput extends UnresolvedInput { + + public AttributeInput(Expression expression, Attribute context) { + super(expression, context); + } + +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/BinaryProcessorDefinition.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/BinaryProcessorDefinition.java new file mode 100644 index 00000000000..2614698f5cf --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/BinaryProcessorDefinition.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition; + +import org.elasticsearch.xpack.sql.expression.Expression; + +import java.util.Arrays; + +public abstract class BinaryProcessorDefinition extends ProcessorDefinition { + + private final ProcessorDefinition left, right; + + public BinaryProcessorDefinition(Expression expression, ProcessorDefinition left, ProcessorDefinition right) { + super(expression, Arrays.asList(left, right)); + this.left = left; + this.right = right; + } + + public ProcessorDefinition left() { + return left; + } + + public ProcessorDefinition right() { + return right; + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ConstantInput.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ConstantInput.java new file mode 100644 index 00000000000..e6c7a5e4494 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ConstantInput.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.ConstantProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; + +public class ConstantInput extends LeafInput { + + public ConstantInput(Expression expression, Object context) { + super(expression, context); + } + + @Override + public Processor asProcessor() { + return new ConstantProcessor(context()); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/HitExtractorInput.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/HitExtractorInput.java new file mode 100644 index 00000000000..5b4dedf3f50 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/HitExtractorInput.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition; + +import org.elasticsearch.xpack.sql.execution.search.extractor.HitExtractor; +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.HitExtractorProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; + +public class HitExtractorInput extends LeafInput { + + public HitExtractorInput(Expression expression, HitExtractor context) { + super(expression, context); + } + + @Override + public Processor asProcessor() { + return new HitExtractorProcessor(context()); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/LeafInput.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/LeafInput.java new file mode 100644 index 00000000000..482049bfbdb --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/LeafInput.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition; + +import org.elasticsearch.xpack.sql.expression.Expression; + +import java.util.Objects; + +import static java.util.Collections.emptyList; + +public abstract class LeafInput extends ProcessorDefinition { + + private T context; + + public LeafInput(Expression expression, T context) { + super(expression, emptyList()); + this.context = context; + } + + public T context() { + return context; + } + + @Override + public int hashCode() { + return Objects.hash(expression(), context); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + AggPathInput other = (AggPathInput) obj; + return Objects.equals(context(), other.context()) + && Objects.equals(expression(), other.expression()); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ProcessorDefinition.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ProcessorDefinition.java new file mode 100644 index 00000000000..5c0f40d6a00 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ProcessorDefinition.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; +import org.elasticsearch.xpack.sql.tree.Node; + +import java.util.List; + +public abstract class ProcessorDefinition extends Node { + + private final Expression expression; + + public ProcessorDefinition(Expression expression, List children) { + super(children); + this.expression = expression; + } + + public Expression expression() { + return expression; + } + + public abstract Processor asProcessor(); +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ProcessorDefinitions.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ProcessorDefinitions.java new file mode 100644 index 00000000000..80c240756ae --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ProcessorDefinitions.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition; + +import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.NamedExpression; +import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction; + +public abstract class ProcessorDefinitions { + + public static ProcessorDefinition toProcessorDefinition(Expression ex) { + if (ex.foldable()) { + return new ConstantInput(ex, ex.fold()); + } + if (ex instanceof ScalarFunction) { + return ((ScalarFunction) ex).asProcessor(); + } + if (ex instanceof AggregateFunction) { + // unresolved AggInput (should always get replaced by the folder) + return new AggPathInput(ex, ((AggregateFunction) ex).name()); + } + if (ex instanceof NamedExpression) { + return new AttributeInput(ex, ((NamedExpression) ex).toAttribute()); + } + throw new SqlIllegalArgumentException("Cannot extract processor from %s", ex); + } +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ReferenceInput.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ReferenceInput.java new file mode 100644 index 00000000000..123718265fc --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/ReferenceInput.java @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.querydsl.container.ColumnReference; + +public class ReferenceInput extends UnresolvedInput { + + public ReferenceInput(Expression expression, ColumnReference context) { + super(expression, context); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/UnaryProcessorDefinition.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/UnaryProcessorDefinition.java new file mode 100644 index 00000000000..8b67e7a7284 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/UnaryProcessorDefinition.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.ChainingProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; + +import java.util.Objects; + +import static java.util.Collections.singletonList; + +public class UnaryProcessorDefinition extends ProcessorDefinition { + + private final ProcessorDefinition child; + private final Processor action; + + public UnaryProcessorDefinition(Expression expression, ProcessorDefinition child, Processor action) { + super(expression, singletonList(child)); + this.child = child; + this.action = action; + } + + public ProcessorDefinition child() { + return child; + } + + public Processor action() { + return action; + } + + @Override + public Processor asProcessor() { + return new ChainingProcessor(child.asProcessor(), action); + } + + @Override + public int hashCode() { + return Objects.hash(expression(), child, action); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + UnaryProcessorDefinition other = (UnaryProcessorDefinition) obj; + return Objects.equals(action, other.action) + && Objects.equals(child, other.child) + && Objects.equals(expression(), other.expression()); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/UnresolvedInput.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/UnresolvedInput.java new file mode 100644 index 00000000000..1ab287718ed --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/definition/UnresolvedInput.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition; + +import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; + +public class UnresolvedInput extends LeafInput { + + UnresolvedInput(Expression expression, T context) { + super(expression, context); + } + + @Override + public Processor asProcessor() { + throw new SqlIllegalArgumentException("Unresolved input - needs resolving first"); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/BinaryProcessor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/BinaryProcessor.java new file mode 100644 index 00000000000..81795923915 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/BinaryProcessor.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +public abstract class BinaryProcessor implements Processor { + + private final Processor left, right; + + public BinaryProcessor(Processor left, Processor right) { + this.left = left; + this.right = right; + } + + protected BinaryProcessor(StreamInput in) throws IOException { + left = in.readNamedWriteable(Processor.class); + right = in.readNamedWriteable(Processor.class); + } + + @Override + public final void writeTo(StreamOutput out) throws IOException { + out.writeNamedWriteable(left); + out.writeNamedWriteable(right); + doWrite(out); + } + + protected abstract void doWrite(StreamOutput out) throws IOException; + + @Override + public Object process(Object input) { + return doProcess(left.process(input), right.process(input)); + } + + protected Processor left() { + return left; + } + + protected Processor right() { + return right; + } + + protected abstract Object doProcess(Object left, Object right); +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/ChainingProcessor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/ChainingProcessor.java new file mode 100644 index 00000000000..9be7de637e3 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/ChainingProcessor.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Objects; + +/** + * A {@linkplain Processor} that composes the results of two + * {@linkplain Processor}s. + */ +public class ChainingProcessor extends UnaryProcessor { + public static final String NAME = "."; + + private final Processor processor; + + public ChainingProcessor(Processor first, Processor second) { + super(first); + this.processor = second; + } + + public ChainingProcessor(StreamInput in) throws IOException { + super(in); + processor = in.readNamedWriteable(Processor.class); + } + + @Override + protected void doWrite(StreamOutput out) throws IOException { + out.writeNamedWriteable(processor); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + protected Object doProcess(Object input) { + return processor.process(input); + } + + Processor first() { + return child(); + } + + Processor second() { + return processor; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), processor); + } + + @Override + public boolean equals(Object obj) { + return super.equals(obj) && Objects.equals(processor, ((ChainingProcessor) obj).processor); + } + + @Override + public String toString() { + return processor + "(" + super.toString() + ")"; + } +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/ConstantProcessor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/ConstantProcessor.java new file mode 100644 index 00000000000..cc419f3c7b7 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/ConstantProcessor.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Objects; + +public class ConstantProcessor implements Processor { + + public static String NAME = "c"; + + private final Object constant; + + public ConstantProcessor(Object value) { + this.constant = value; + } + + public ConstantProcessor(StreamInput in) throws IOException { + constant = in.readGenericValue(); + } + + public void writeTo(StreamOutput out) throws IOException { + out.writeGenericValue(constant); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public Object process(Object input) { + return constant; + } + + @Override + public int hashCode() { + return Objects.hashCode(constant); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + ConstantProcessor other = (ConstantProcessor) obj; + return Objects.equals(constant, other.constant); + } + + @Override + public String toString() { + return "^" + constant; + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/HitExtractorProcessor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/HitExtractorProcessor.java new file mode 100644 index 00000000000..898d54a1e2d --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/HitExtractorProcessor.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; +import org.elasticsearch.xpack.sql.execution.search.extractor.HitExtractor; + +import java.io.IOException; +import java.util.Objects; + +/** + * Processor wrapping a HitExtractor esentially being a source/leaf of a + * Processor tree. + */ +public class HitExtractorProcessor implements Processor { + + public static final String NAME = "h"; + + private final HitExtractor extractor; + + public HitExtractorProcessor(HitExtractor extractor) { + this.extractor = extractor; + } + + public HitExtractorProcessor(StreamInput in) throws IOException { + extractor = in.readNamedWriteable(HitExtractor.class); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + extractor.writeTo(out); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public Object process(Object input) { + if (!(input instanceof SearchHit)) { + throw new SqlIllegalArgumentException("Expected a SearchHit but received %s", input); + } + return extractor.get((SearchHit) input); + } + + @Override + public int hashCode() { + return Objects.hash(extractor); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + HitExtractorProcessor other = (HitExtractorProcessor) obj; + return Objects.equals(extractor, other.extractor); + } + + @Override + public String toString() { + return extractor.toString(); + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/MatrixFieldProcessor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/MatrixFieldProcessor.java similarity index 71% rename from sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/MatrixFieldProcessor.java rename to sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/MatrixFieldProcessor.java index adbce5799ab..7036ca79fb4 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/MatrixFieldProcessor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/MatrixFieldProcessor.java @@ -3,15 +3,16 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.sql.expression.function.scalar; +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; import java.util.Map; +import java.util.Objects; -public class MatrixFieldProcessor implements ColumnProcessor { +public class MatrixFieldProcessor implements Processor { public static final String NAME = "mat"; private final String key; @@ -20,7 +21,7 @@ public class MatrixFieldProcessor implements ColumnProcessor { this.key = key; } - MatrixFieldProcessor(StreamInput in) throws IOException { + public MatrixFieldProcessor(StreamInput in) throws IOException { key = in.readString(); } @@ -39,25 +40,30 @@ public class MatrixFieldProcessor implements ColumnProcessor { } @Override - public Object apply(Object r) { + public Object process(Object r) { return r instanceof Map ? ((Map) r).get(key) : r; } @Override - public boolean equals(Object obj) { - if (obj == null || obj.getClass() != getClass()) { - return false; - } - MatrixFieldProcessor other = (MatrixFieldProcessor) obj; - return key.equals(other.key); + public int hashCode() { + return Objects.hashCode(key); } @Override - public int hashCode() { - return key.hashCode(); + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + MatrixFieldProcessor other = (MatrixFieldProcessor) obj; + return Objects.equals(key, other.key); } public String toString() { return "[" + key + "]"; } -} +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/Reference.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/Processor.java similarity index 51% rename from sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/Reference.java rename to sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/Processor.java index e3b164a8fd7..f77568368b4 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/Reference.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/Processor.java @@ -3,10 +3,11 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.sql.querydsl.container; +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime; -public interface Reference { - default int depth() { - return 0; - } +import org.elasticsearch.common.io.stream.NamedWriteable; + +public interface Processor extends NamedWriteable { + + Object process(Object input); } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/SuppliedProcessor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/SuppliedProcessor.java new file mode 100644 index 00000000000..2abbfd4506f --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/SuppliedProcessor.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime; + +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; + +import java.io.IOException; +import java.util.function.Supplier; + +public class SuppliedProcessor implements Processor { + + private final Supplier action; + + public SuppliedProcessor(Supplier action) { + this.action = action; + } + + @Override + public String getWriteableName() { + throw new SqlIllegalArgumentException("transient"); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + throw new SqlIllegalArgumentException("transient"); + } + + @Override + public Object process(Object input) { + return action.get(); + } + +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/UnaryProcessor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/UnaryProcessor.java new file mode 100644 index 00000000000..613e2632283 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/UnaryProcessor.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Objects; + +public abstract class UnaryProcessor implements Processor { + + private final Processor child; + + public UnaryProcessor(Processor child) { + this.child = child; + } + + protected UnaryProcessor(StreamInput in) throws IOException { + child = in.readNamedWriteable(Processor.class); + } + + @Override + public final void writeTo(StreamOutput out) throws IOException { + out.writeNamedWriteable(child); + doWrite(out); + } + + protected abstract void doWrite(StreamOutput out) throws IOException; + + @Override + public final Object process(Object input) { + return doProcess(child.process(input)); + } + + public Processor child() { + return child; + } + + protected abstract Object doProcess(Object input); + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + UnaryProcessor other = (UnaryProcessor) obj; + return Objects.equals(child, other.child); + } + + @Override + public int hashCode() { + return Objects.hashCode(child); + } + + @Override + public String toString() { + return Objects.toString(child); + } +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/script/ScriptTemplate.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/script/ScriptTemplate.java index 3cd5d6c84eb..2939df826f0 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/script/ScriptTemplate.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/script/ScriptTemplate.java @@ -5,16 +5,16 @@ */ package org.elasticsearch.xpack.sql.expression.function.scalar.script; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; - import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.type.DataTypes; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; + import static java.lang.String.format; public class ScriptTemplate { @@ -60,7 +60,7 @@ public class ScriptTemplate { private String bindTemplate() { List binding = params.asCodeNames(); - return binding.isEmpty() ? template : String.format(Locale.ROOT, template, binding.toArray()); + return binding.isEmpty() ? template : format(Locale.ROOT, template, binding.toArray()); } @Override diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/And.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/And.java index 7747477b27a..39a076ce491 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/And.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/And.java @@ -5,12 +5,12 @@ */ package org.elasticsearch.xpack.sql.expression.predicate; -import org.elasticsearch.xpack.sql.expression.BinaryExpression; +import org.elasticsearch.xpack.sql.expression.BinaryLogic; +import org.elasticsearch.xpack.sql.expression.BinaryOperator.Negateable; import org.elasticsearch.xpack.sql.expression.Expression; -import org.elasticsearch.xpack.sql.expression.BinaryExpression.Negateable; import org.elasticsearch.xpack.sql.tree.Location; -public class And extends BinaryExpression implements Negateable { +public class And extends BinaryLogic implements Negateable { public And(Location location, Expression left, Expression right) { super(location, left, right); diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/BinaryComparison.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/BinaryComparison.java index 52bcd9bf689..25aa003c7d2 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/BinaryComparison.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/BinaryComparison.java @@ -5,19 +5,31 @@ */ package org.elasticsearch.xpack.sql.expression.predicate; -import org.elasticsearch.xpack.sql.expression.BinaryExpression; +import org.elasticsearch.xpack.sql.expression.BinaryOperator; import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DataTypes; // marker class to indicate operations that rely on values -public abstract class BinaryComparison extends BinaryExpression { +public abstract class BinaryComparison extends BinaryOperator { public BinaryComparison(Location location, Expression left, Expression right) { super(location, left, right); } + @Override + protected TypeResolution resolveInputType(DataType inputType) { + return TypeResolution.TYPE_RESOLVED; + } + @Override protected Expression canonicalize() { return left().hashCode() > right().hashCode() ? swapLeftAndRight() : this; } + + @Override + public DataType dataType() { + return DataTypes.BOOLEAN; + } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/GreaterThan.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/GreaterThan.java index 4b48612d163..ffc1c89e6d4 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/GreaterThan.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/GreaterThan.java @@ -5,8 +5,8 @@ */ package org.elasticsearch.xpack.sql.expression.predicate; +import org.elasticsearch.xpack.sql.expression.BinaryOperator.Negateable; import org.elasticsearch.xpack.sql.expression.Expression; -import org.elasticsearch.xpack.sql.expression.BinaryExpression.Negateable; import org.elasticsearch.xpack.sql.tree.Location; public class GreaterThan extends BinaryComparison implements Negateable { diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/GreaterThanOrEqual.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/GreaterThanOrEqual.java index 1ac4b0fa0d1..475afe66310 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/GreaterThanOrEqual.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/GreaterThanOrEqual.java @@ -5,8 +5,8 @@ */ package org.elasticsearch.xpack.sql.expression.predicate; +import org.elasticsearch.xpack.sql.expression.BinaryOperator.Negateable; import org.elasticsearch.xpack.sql.expression.Expression; -import org.elasticsearch.xpack.sql.expression.BinaryExpression.Negateable; import org.elasticsearch.xpack.sql.tree.Location; public class GreaterThanOrEqual extends BinaryComparison implements Negateable { diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/LessThan.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/LessThan.java index 824d695493c..05286ff9016 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/LessThan.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/LessThan.java @@ -5,8 +5,8 @@ */ package org.elasticsearch.xpack.sql.expression.predicate; +import org.elasticsearch.xpack.sql.expression.BinaryOperator.Negateable; import org.elasticsearch.xpack.sql.expression.Expression; -import org.elasticsearch.xpack.sql.expression.BinaryExpression.Negateable; import org.elasticsearch.xpack.sql.tree.Location; public class LessThan extends BinaryComparison implements Negateable { diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/LessThanOrEqual.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/LessThanOrEqual.java index a3d46c93e1a..dc61e0362e9 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/LessThanOrEqual.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/LessThanOrEqual.java @@ -5,8 +5,8 @@ */ package org.elasticsearch.xpack.sql.expression.predicate; +import org.elasticsearch.xpack.sql.expression.BinaryOperator.Negateable; import org.elasticsearch.xpack.sql.expression.Expression; -import org.elasticsearch.xpack.sql.expression.BinaryExpression.Negateable; import org.elasticsearch.xpack.sql.tree.Location; public class LessThanOrEqual extends BinaryComparison implements Negateable { diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/Not.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/Not.java index fc1cfc72321..2b8470e86d3 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/Not.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/Not.java @@ -5,9 +5,9 @@ */ package org.elasticsearch.xpack.sql.expression.predicate; +import org.elasticsearch.xpack.sql.expression.BinaryOperator.Negateable; import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.expression.UnaryExpression; -import org.elasticsearch.xpack.sql.expression.BinaryExpression.Negateable; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.type.DataTypes; diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/Or.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/Or.java index 1b624b43cde..1d45d186e62 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/Or.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/Or.java @@ -5,12 +5,12 @@ */ package org.elasticsearch.xpack.sql.expression.predicate; -import org.elasticsearch.xpack.sql.expression.BinaryExpression; +import org.elasticsearch.xpack.sql.expression.BinaryLogic; +import org.elasticsearch.xpack.sql.expression.BinaryOperator.Negateable; import org.elasticsearch.xpack.sql.expression.Expression; -import org.elasticsearch.xpack.sql.expression.BinaryExpression.Negateable; import org.elasticsearch.xpack.sql.tree.Location; -public class Or extends BinaryExpression implements Negateable { +public class Or extends BinaryLogic implements Negateable { public Or(Location location, Expression left, Expression right) { super(location, left, right); diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/regex/Like.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/regex/Like.java index f7a97396706..77ec997e055 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/regex/Like.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/regex/Like.java @@ -8,6 +8,8 @@ package org.elasticsearch.xpack.sql.expression.regex; import org.elasticsearch.xpack.sql.expression.BinaryExpression; import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DataTypes; public class Like extends BinaryExpression { @@ -20,9 +22,13 @@ public class Like extends BinaryExpression { return new Like(location(), right(), left()); } + @Override + public DataType dataType() { + return DataTypes.BOOLEAN; + } @Override public String symbol() { - return " LIKE "; + return "LIKE"; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/regex/RLike.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/regex/RLike.java index 50d613664e2..acdaf636a99 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/regex/RLike.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/regex/RLike.java @@ -8,6 +8,8 @@ package org.elasticsearch.xpack.sql.expression.regex; import org.elasticsearch.xpack.sql.expression.BinaryExpression; import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DataTypes; public class RLike extends BinaryExpression { @@ -20,8 +22,13 @@ public class RLike extends BinaryExpression { return new RLike(location(), right(), left()); } + @Override + public DataType dataType() { + return DataTypes.BOOLEAN; + } + @Override public String symbol() { - return " RLIKE "; + return "RLIKE"; } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java index dc89c6288cf..fd18931db6d 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java @@ -10,7 +10,7 @@ import org.elasticsearch.xpack.sql.expression.Alias; import org.elasticsearch.xpack.sql.expression.Attribute; import org.elasticsearch.xpack.sql.expression.AttributeSet; import org.elasticsearch.xpack.sql.expression.BinaryExpression; -import org.elasticsearch.xpack.sql.expression.BinaryExpression.Negateable; +import org.elasticsearch.xpack.sql.expression.BinaryOperator.Negateable; import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.expression.ExpressionSet; import org.elasticsearch.xpack.sql.expression.Expressions; @@ -45,14 +45,15 @@ import org.elasticsearch.xpack.sql.expression.predicate.Range; import org.elasticsearch.xpack.sql.plan.logical.Aggregate; import org.elasticsearch.xpack.sql.plan.logical.Filter; import org.elasticsearch.xpack.sql.plan.logical.Limit; +import org.elasticsearch.xpack.sql.plan.logical.LocalRelation; import org.elasticsearch.xpack.sql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.sql.plan.logical.OrderBy; import org.elasticsearch.xpack.sql.plan.logical.Project; -import org.elasticsearch.xpack.sql.plan.logical.Queryless; import org.elasticsearch.xpack.sql.plan.logical.SubQueryAlias; import org.elasticsearch.xpack.sql.rule.Rule; import org.elasticsearch.xpack.sql.rule.RuleExecutor; import org.elasticsearch.xpack.sql.session.EmptyExecutable; +import org.elasticsearch.xpack.sql.session.SingletonExecutable; import java.util.ArrayList; import java.util.Arrays; @@ -63,6 +64,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import static java.util.stream.Collectors.toList; import static org.elasticsearch.xpack.sql.expression.Literal.FALSE; @@ -89,7 +91,7 @@ public class Optimizer extends RuleExecutor { protected Iterable.Batch> batches() { Batch resolution = new Batch("Finish Analysis", new PruneSubqueryAliases(), - new CleanAliases() + CleanAliases.INSTANCE ); Batch aggregate = new Batch("Aggregation", @@ -103,32 +105,42 @@ public class Optimizer extends RuleExecutor { new CombineAggsToPercentileRanks() ); - Batch cleanup = new Batch("Operator Optimization", + Batch operators = new Batch("Operator Optimization", // can't really do it since alias information is lost (packed inside EsQuery) //new ProjectPruning(), + + // folding + new ReplaceFoldableAttributes(), + new ConstantFolding(), + // boolean new BooleanSimplification(), new BinaryComparisonSimplification(), new BooleanLiteralsOnTheRight(), new CombineComparisonsIntoRange(), + // prune/elimination new PruneFilters(), new PruneOrderBy(), new PruneOrderByNestedFields(), new PruneCast(), - new PruneDuplicateFunctions(), - new SkipQueryOnLimitZero() + new PruneDuplicateFunctions() + ); + + Batch local = new Batch("Skip Elasticsearch", + new SkipQueryOnLimitZero(), + new SkipQueryIfFoldingProjection() ); //new BalanceBooleanTrees()); Batch label = new Batch("Set as Optimized", Limiter.ONCE, new SetAsOptimized()); - return Arrays.asList(resolution, aggregate, cleanup, label); + return Arrays.asList(resolution, aggregate, operators, local, label); } static class PruneSubqueryAliases extends OptimizerRule { PruneSubqueryAliases() { - super(false); + super(TransformDirection.UP); } @Override @@ -139,8 +151,10 @@ public class Optimizer extends RuleExecutor { static class CleanAliases extends OptimizerRule { + private static final CleanAliases INSTANCE = new CleanAliases(); + CleanAliases() { - super(false); + super(TransformDirection.UP); } @Override @@ -555,7 +569,7 @@ public class Optimizer extends RuleExecutor { } // TODO: add comparison with null as well if (FALSE.equals(filter.condition())) { - throw new UnsupportedOperationException("Put empty relation"); + return new LocalRelation(filter.location(), new EmptyExecutable(filter.output())); } } @@ -736,7 +750,7 @@ public class Optimizer extends RuleExecutor { Cast c = (Cast) as.child(); if (c.from().same(c.to())) { - Alias newAs = new Alias(as.location(), as.name(), as.qualifier(), c.argument(), as.id(), as.synthetic()); + Alias newAs = new Alias(as.location(), as.name(), as.qualifier(), c.field(), as.id(), as.synthetic()); replacedCast.put(as.toAttribute(), newAs.toAttribute()); return newAs; } @@ -747,16 +761,16 @@ public class Optimizer extends RuleExecutor { }); // then handle stand-alone casts (mixed together the cast rule will kick in before the alias) - transformed = plan.transformExpressionsUp(e -> { + transformed = transformed.transformExpressionsUp(e -> { if (e instanceof Cast) { Cast c = (Cast) e; if (c.from().same(c.to())) { - Expression argument = c.argument(); - if (!(argument instanceof NamedExpression)) { - throw new SqlIllegalArgumentException("Expected a NamedExpression but got %s", argument); + Expression argument = c.field(); + if (argument instanceof NamedExpression) { + replacedCast.put(c.toAttribute(), ((NamedExpression) argument).toAttribute()); } - replacedCast.put(c.toAttribute(), ((NamedExpression) argument).toAttribute()); + return argument; } } @@ -817,19 +831,6 @@ public class Optimizer extends RuleExecutor { } } - static class SkipQueryOnLimitZero extends OptimizerRule { - - @Override - protected LogicalPlan rule(Limit limit) { - if (limit.limit() instanceof Literal) { - if (Integer.valueOf(0).equals(Integer.parseInt(((Literal) limit.limit()).value().toString()))) { - return new Queryless(limit.location(), new EmptyExecutable(limit.output())); - } - } - return limit; - } - } - static class CombineFilters extends OptimizerRule { @Override @@ -842,7 +843,118 @@ public class Optimizer extends RuleExecutor { } } - static class BooleanSimplification extends OptimizerExpressionUpRule { + + // replace attributes of foldable expressions with the foldable trees + // SELECT 5 a, a + 3, ... + + static class ReplaceFoldableAttributes extends Rule { + + @Override + public LogicalPlan apply(LogicalPlan plan) { + return rule(plan); + } + + @Override + protected LogicalPlan rule(LogicalPlan plan) { + Map aliases = new LinkedHashMap<>(); + List attrs = new ArrayList<>(); + + // find aliases of all projections + plan.forEachDown(p -> { + for (NamedExpression ne : p.projections()) { + if (ne instanceof Alias) { + if (((Alias) ne).child().foldable()) { + Attribute attr = ne.toAttribute(); + attrs.add(attr); + aliases.put(attr, (Alias) ne); + } + } + } + }, Project.class); + + if (attrs.isEmpty()) { + return plan; + } + + AtomicBoolean stop = new AtomicBoolean(false); + + // propagate folding up to unary nodes + // anything higher and the propagate stops + plan = plan.transformUp(p -> { + if (stop.get() == false && canPropagateFoldable(p)) { + return p.transformExpressionsDown(e -> { + if (e instanceof Attribute && attrs.contains(e)) { + Alias as = aliases.get(e); + if (as == null) { + throw new SqlIllegalArgumentException("need to implement AttributeMap"); + } + return as; + } + return e; + }); + } + + if (p.children().size() > 1) { + stop.set(true); + } + + return p; + }); + + // finally clean-up aliases + return CleanAliases.INSTANCE.apply(plan); + + } + + private boolean canPropagateFoldable(LogicalPlan p) { + return p instanceof Project || p instanceof Filter || p instanceof SubQueryAlias || p instanceof Aggregate || p instanceof Limit || p instanceof OrderBy; + } + } + + static class ConstantFolding extends OptimizerExpressionRule { + + ConstantFolding() { + super(TransformDirection.DOWN); + } + + @Override + protected Expression rule(Expression e) { + // preserve aliases + if (e instanceof Alias) { + Alias a = (Alias) e; + Expression fold = fold(a.child()); + if (fold != e) { + return new Alias(a.location(), a.name(), null, fold, a.id()); + } + return a; + } + + Expression fold = fold(e); + if (fold != e) { + // preserve the name through an alias + if (e instanceof NamedExpression) { + NamedExpression ne = (NamedExpression) e; + return new Alias(e.location(), ne.name(), null, fold, ne.id()); + } + return fold; + } + return e; + } + + private Expression fold(Expression e) { + // literals are always foldable, so avoid creating a duplicate + if (e.foldable() && !(e instanceof Literal)) { + return new Literal(e.location(), e.fold(), e.dataType()); + } + return e; + } + } + + static class BooleanSimplification extends OptimizerExpressionRule { + + BooleanSimplification() { + super(TransformDirection.UP); + } @Override protected Expression rule(Expression e) { @@ -961,7 +1073,11 @@ public class Optimizer extends RuleExecutor { } } - static class BinaryComparisonSimplification extends OptimizerExpressionUpRule { + static class BinaryComparisonSimplification extends OptimizerExpressionRule { + + BinaryComparisonSimplification() { + super(TransformDirection.UP); + } @Override protected Expression rule(Expression e) { @@ -990,7 +1106,11 @@ public class Optimizer extends RuleExecutor { } } - static class BooleanLiteralsOnTheRight extends OptimizerExpressionUpRule { + static class BooleanLiteralsOnTheRight extends OptimizerExpressionRule { + + BooleanLiteralsOnTheRight() { + super(TransformDirection.UP); + } @Override protected Expression rule(Expression e) { @@ -1002,7 +1122,11 @@ public class Optimizer extends RuleExecutor { } } - static class CombineComparisonsIntoRange extends OptimizerExpressionUpRule { + static class CombineComparisonsIntoRange extends OptimizerExpressionRule { + + CombineComparisonsIntoRange() { + super(TransformDirection.UP); + } @Override protected Expression rule(Expression e) { @@ -1040,6 +1164,56 @@ public class Optimizer extends RuleExecutor { } + static class SkipQueryOnLimitZero extends OptimizerRule { + @Override + protected LogicalPlan rule(Limit limit) { + if (limit.limit() instanceof Literal) { + if (Integer.valueOf(0).equals((Number) (((Literal) limit.limit()).fold()))) { + return new LocalRelation(limit.location(), new EmptyExecutable(limit.output())); + } + } + return limit; + } + } + + static class SkipQueryIfFoldingProjection extends OptimizerRule { + @Override + protected LogicalPlan rule(LogicalPlan plan) { + if (plan instanceof Project) { + Project p = (Project) plan; + List values = extractConstants(p.projections()); + if (values.size() == p.projections().size()) { + return new LocalRelation(p.location(), new SingletonExecutable(p.output(), values.toArray())); + } + } + if (plan instanceof Aggregate) { + Aggregate a = (Aggregate) plan; + List values = extractConstants(a.aggregates()); + if (values.size() == a.aggregates().size()) { + return new LocalRelation(a.location(), new SingletonExecutable(a.output(), values.toArray())); + } + } + return plan; + } + + private List extractConstants(List named) { + List values = new ArrayList<>(); + for (NamedExpression n : named) { + if (n instanceof Alias) { + Alias a = (Alias) n; + if (a.child().foldable()) { + values.add(a.child().fold()); + } + else { + return values; + } + } + } + return values; + } + } + + static class SetAsOptimized extends Rule { @Override @@ -1060,43 +1234,38 @@ public class Optimizer extends RuleExecutor { abstract static class OptimizerRule extends Rule { - private final boolean transformDown; + private final TransformDirection direction; OptimizerRule() { - this(true); + this(TransformDirection.DOWN); } - OptimizerRule(boolean transformDown) { - this.transformDown = transformDown; + OptimizerRule(TransformDirection direction) { + this.direction = direction; } @Override public final LogicalPlan apply(LogicalPlan plan) { - return transformDown ? plan.transformDown(this::rule, typeToken()) : plan.transformUp(this::rule, typeToken()); + return direction == TransformDirection.DOWN ? plan.transformDown(this::rule, typeToken()) : plan.transformUp(this::rule, typeToken()); } @Override protected abstract LogicalPlan rule(SubPlan plan); } - abstract static class OptimizerExpressionUpRule extends Rule { + abstract static class OptimizerExpressionRule extends Rule { - private final boolean transformDown; + private final TransformDirection direction; - OptimizerExpressionUpRule() { - //NB: expressions are transformed up (not down like the plan) - this(false); + OptimizerExpressionRule(TransformDirection direction) { + this.direction = direction; } - OptimizerExpressionUpRule(boolean transformDown) { - this.transformDown = transformDown; - } - - @Override public final LogicalPlan apply(LogicalPlan plan) { - return transformDown ? plan.transformExpressionsDown(this::rule) : plan.transformExpressionsUp(this::rule); + return direction == TransformDirection.DOWN ? plan.transformExpressionsDown(this::rule) : plan + .transformExpressionsUp(this::rule); } @Override @@ -1106,4 +1275,8 @@ public class Optimizer extends RuleExecutor { protected abstract Expression rule(Expression e); } + + enum TransformDirection { + UP, DOWN + }; } \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java index 16f46518dca..baaa6aa9674 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java @@ -19,6 +19,12 @@ import org.elasticsearch.xpack.sql.expression.UnresolvedAttribute; import org.elasticsearch.xpack.sql.expression.UnresolvedStar; import org.elasticsearch.xpack.sql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.sql.expression.function.scalar.Cast; +import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.Add; +import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.Div; +import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.Mod; +import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.Mul; +import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.Sub; +import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.Neg; import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.Extract; import org.elasticsearch.xpack.sql.expression.predicate.And; import org.elasticsearch.xpack.sql.expression.predicate.Equals; @@ -36,6 +42,8 @@ import org.elasticsearch.xpack.sql.expression.predicate.fulltext.MultiMatchQuery import org.elasticsearch.xpack.sql.expression.predicate.fulltext.StringQueryPredicate; import org.elasticsearch.xpack.sql.expression.regex.Like; import org.elasticsearch.xpack.sql.expression.regex.RLike; +import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ArithmeticBinaryContext; +import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ArithmeticUnaryContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.BooleanLiteralContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.CastContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ColumnExpressionContext; @@ -211,10 +219,51 @@ abstract class ExpressionBuilder extends IdentifierBuilder { return pCtx.NOT() != null ? new Not(loc, e) : e; } + // + // Arithmetic + // + + @Override + public Object visitArithmeticUnary(ArithmeticUnaryContext ctx) { + Expression value = expression(ctx.valueExpression()); + Location loc = source(ctx); + + switch (ctx.operator.getType()) { + case SqlBaseParser.PLUS: + return value; + case SqlBaseParser.MINUS: + return new Neg(source(ctx.operator), value); + default: + throw new ParsingException(loc, "Unknown arithemtic %s", ctx.operator.getText()); + } + } + + @Override + public Object visitArithmeticBinary(ArithmeticBinaryContext ctx) { + Expression left = expression(ctx.left); + Expression right = expression(ctx.right); + + Location loc = source(ctx.operator); + + switch (ctx.operator.getType()) { + case SqlBaseParser.ASTERISK: + return new Mul(loc, left, right); + case SqlBaseParser.SLASH: + return new Div(loc, left, right); + case SqlBaseParser.PERCENT: + return new Mod(loc, left, right); + case SqlBaseParser.PLUS: + return new Add(loc, left, right); + case SqlBaseParser.MINUS: + return new Sub(loc, left, right); + default: + throw new ParsingException(loc, "Unknown arithemtic %s", ctx.operator.getText()); + } + } + // // Full-text search predicates // - @Override public Object visitStringQuery(StringQueryContext ctx) { return new StringQueryPredicate(source(ctx), text(ctx.queryString), text(ctx.options)); diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/parser/LogicalPlanBuilder.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/parser/LogicalPlanBuilder.java index 33adbe76add..1cd09b30823 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/parser/LogicalPlanBuilder.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/parser/LogicalPlanBuilder.java @@ -27,7 +27,7 @@ import org.elasticsearch.xpack.sql.plan.TableIdentifier; import org.elasticsearch.xpack.sql.plan.logical.Aggregate; import org.elasticsearch.xpack.sql.plan.logical.Distinct; import org.elasticsearch.xpack.sql.plan.logical.Filter; -import org.elasticsearch.xpack.sql.plan.logical.FromlessSelect; +import org.elasticsearch.xpack.sql.plan.logical.LocalRelation; import org.elasticsearch.xpack.sql.plan.logical.Join; import org.elasticsearch.xpack.sql.plan.logical.Join.JoinType; import org.elasticsearch.xpack.sql.plan.logical.Limit; @@ -37,6 +37,7 @@ import org.elasticsearch.xpack.sql.plan.logical.Project; import org.elasticsearch.xpack.sql.plan.logical.SubQueryAlias; import org.elasticsearch.xpack.sql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.sql.plan.logical.With; +import org.elasticsearch.xpack.sql.session.EmptyExecutable; import org.elasticsearch.xpack.sql.type.DataTypes; import java.util.LinkedHashMap; @@ -87,7 +88,7 @@ abstract class LogicalPlanBuilder extends ExpressionBuilder { @Override public LogicalPlan visitQuerySpecification(QuerySpecificationContext ctx) { - LogicalPlan query = (ctx.fromClause() != null)? plan(ctx.fromClause()) : new FromlessSelect(source(ctx)); + LogicalPlan query = (ctx.fromClause() != null)? plan(ctx.fromClause()) : new LocalRelation(source(ctx), new EmptyExecutable(emptyList())); // add WHERE if (ctx.where != null) { diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/CatalogTable.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/EsRelation.java similarity index 95% rename from sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/CatalogTable.java rename to sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/EsRelation.java index 766b7a18768..f53b6c0d56c 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/CatalogTable.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/EsRelation.java @@ -25,12 +25,12 @@ import static java.util.Collections.emptyList; import static java.util.stream.Collectors.toList; import static org.elasticsearch.xpack.sql.util.CollectionUtils.combine; -public class CatalogTable extends LeafPlan { +public class EsRelation extends LeafPlan { private final EsIndex index; private final List attrs; - public CatalogTable(Location location, EsIndex index) { + public EsRelation(Location location, EsIndex index) { super(location); this.index = index; attrs = flatten(location, index.mapping()).collect(toList()); @@ -90,7 +90,7 @@ public class CatalogTable extends LeafPlan { return false; } - CatalogTable other = (CatalogTable) obj; + EsRelation other = (EsRelation) obj; return Objects.equals(index, other.index); } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/FromlessSelect.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/FromlessSelect.java deleted file mode 100644 index 1caf9ebe19b..00000000000 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/FromlessSelect.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.sql.plan.logical; - -import java.util.List; - -import org.elasticsearch.xpack.sql.expression.Attribute; -import org.elasticsearch.xpack.sql.tree.Location; - -import static java.util.Collections.emptyList; - -/** - * Special class when using SELECT without a FROM. - * Rarely the case however it can be used be used as an alternative to the 'DUAL' table in Orace. - */ -public class FromlessSelect extends LeafPlan { - - public FromlessSelect(Location location) { - super(location); - } - - @Override - public List output() { - return emptyList(); - } - - @Override - public boolean expressionsResolved() { - return true; - } - - @Override - public int hashCode() { - return FromlessSelect.class.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - return true; - } -} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/Queryless.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/LocalRelation.java similarity index 89% rename from sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/Queryless.java rename to sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/LocalRelation.java index db2f4d33094..a8bb2665d7e 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/Queryless.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/LocalRelation.java @@ -5,9 +5,6 @@ */ package org.elasticsearch.xpack.sql.plan.logical; -import java.util.List; -import java.util.Objects; - import org.elasticsearch.action.ActionListener; import org.elasticsearch.xpack.sql.expression.Attribute; import org.elasticsearch.xpack.sql.session.Executable; @@ -15,13 +12,16 @@ import org.elasticsearch.xpack.sql.session.RowSetCursor; import org.elasticsearch.xpack.sql.session.SqlSession; import org.elasticsearch.xpack.sql.tree.Location; +import java.util.List; +import java.util.Objects; + import static java.util.Collections.emptyList; -public class Queryless extends LogicalPlan implements Executable { +public class LocalRelation extends LogicalPlan implements Executable { private final Executable executable; - public Queryless(Location location, Executable executable) { + public LocalRelation(Location location, Executable executable) { super(location, emptyList()); this.executable = executable; } @@ -60,7 +60,7 @@ public class Queryless extends LogicalPlan implements Executable { return false; } - Queryless other = (Queryless) obj; + LocalRelation other = (LocalRelation) obj; return Objects.equals(executable, other.executable); } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/LogicalPlan.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/LogicalPlan.java index c1b07fe6081..5bf364d9ef7 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/LogicalPlan.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/logical/LogicalPlan.java @@ -5,13 +5,13 @@ */ package org.elasticsearch.xpack.sql.plan.logical; -import java.util.List; - import org.elasticsearch.xpack.sql.capabilities.Resolvable; import org.elasticsearch.xpack.sql.capabilities.Resolvables; import org.elasticsearch.xpack.sql.plan.QueryPlan; import org.elasticsearch.xpack.sql.tree.Location; +import java.util.List; + public abstract class LogicalPlan extends QueryPlan implements Resolvable { private boolean analyzed = false; @@ -62,4 +62,4 @@ public abstract class LogicalPlan extends QueryPlan implements Reso @Override public abstract boolean equals(Object obj); -} +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/physical/QuerylessExec.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/physical/LocalExec.java similarity index 82% rename from sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/physical/QuerylessExec.java rename to sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/physical/LocalExec.java index e808743a6df..4fa4fa0b834 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/physical/QuerylessExec.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/plan/physical/LocalExec.java @@ -5,21 +5,22 @@ */ package org.elasticsearch.xpack.sql.plan.physical; -import java.util.List; -import java.util.Objects; - import org.elasticsearch.action.ActionListener; import org.elasticsearch.xpack.sql.expression.Attribute; +import org.elasticsearch.xpack.sql.session.EmptyExecutable; import org.elasticsearch.xpack.sql.session.Executable; import org.elasticsearch.xpack.sql.session.RowSetCursor; import org.elasticsearch.xpack.sql.session.SqlSession; import org.elasticsearch.xpack.sql.tree.Location; -public class QuerylessExec extends LeafExec { +import java.util.List; +import java.util.Objects; + +public class LocalExec extends LeafExec { private final Executable executable; - public QuerylessExec(Location location, Executable executable) { + public LocalExec(Location location, Executable executable) { super(location); this.executable = executable; } @@ -33,6 +34,10 @@ public class QuerylessExec extends LeafExec { return executable.output(); } + public boolean isEmpty() { + return executable instanceof EmptyExecutable; + } + @Override public void execute(SqlSession session, ActionListener listener) { executable.execute(session, listener); @@ -53,7 +58,7 @@ public class QuerylessExec extends LeafExec { return false; } - QuerylessExec other = (QuerylessExec) obj; + LocalExec other = (LocalExec) obj; return Objects.equals(executable, other.executable); } } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/Mapper.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/Mapper.java index 2d36e250437..caa0734ce37 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/Mapper.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/Mapper.java @@ -7,14 +7,14 @@ package org.elasticsearch.xpack.sql.planner; import org.elasticsearch.xpack.sql.expression.Attribute; import org.elasticsearch.xpack.sql.plan.logical.Aggregate; -import org.elasticsearch.xpack.sql.plan.logical.CatalogTable; +import org.elasticsearch.xpack.sql.plan.logical.EsRelation; import org.elasticsearch.xpack.sql.plan.logical.Filter; import org.elasticsearch.xpack.sql.plan.logical.Join; import org.elasticsearch.xpack.sql.plan.logical.Limit; import org.elasticsearch.xpack.sql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.sql.plan.logical.OrderBy; import org.elasticsearch.xpack.sql.plan.logical.Project; -import org.elasticsearch.xpack.sql.plan.logical.Queryless; +import org.elasticsearch.xpack.sql.plan.logical.LocalRelation; import org.elasticsearch.xpack.sql.plan.logical.With; import org.elasticsearch.xpack.sql.plan.logical.command.Command; import org.elasticsearch.xpack.sql.plan.physical.AggregateExec; @@ -25,7 +25,7 @@ import org.elasticsearch.xpack.sql.plan.physical.LimitExec; import org.elasticsearch.xpack.sql.plan.physical.OrderExec; import org.elasticsearch.xpack.sql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.sql.plan.physical.ProjectExec; -import org.elasticsearch.xpack.sql.plan.physical.QuerylessExec; +import org.elasticsearch.xpack.sql.plan.physical.LocalExec; import org.elasticsearch.xpack.sql.plan.physical.UnplannedExec; import org.elasticsearch.xpack.sql.querydsl.container.QueryContainer; import org.elasticsearch.xpack.sql.rule.Rule; @@ -63,8 +63,8 @@ class Mapper extends RuleExecutor { return new CommandExec((Command) p); } - if (p instanceof Queryless) { - return new QuerylessExec(p.location(), (Queryless) p); + if (p instanceof LocalRelation) { + return new LocalExec(p.location(), (LocalRelation) p); } if (p instanceof Project) { @@ -88,8 +88,8 @@ class Mapper extends RuleExecutor { return new AggregateExec(map(a.child()), a.groupings(), a.aggregates()); } - if (p instanceof CatalogTable) { - CatalogTable c = (CatalogTable) p; + if (p instanceof EsRelation) { + EsRelation c = (EsRelation) p; List output = c.output(); return new EsQueryExec(c.location(), c.index().name(), output, new QueryContainer()); } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java index 1a95fcf77d4..53e5e2f1bdb 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java @@ -5,35 +5,35 @@ */ package org.elasticsearch.xpack.sql.planner; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.expression.Alias; import org.elasticsearch.xpack.sql.expression.Attribute; import org.elasticsearch.xpack.sql.expression.Expression; -import org.elasticsearch.xpack.sql.expression.ExpressionId; +import org.elasticsearch.xpack.sql.expression.Expressions; import org.elasticsearch.xpack.sql.expression.Foldables; import org.elasticsearch.xpack.sql.expression.NamedExpression; -import org.elasticsearch.xpack.sql.expression.NestedFieldAttribute; import org.elasticsearch.xpack.sql.expression.Order; -import org.elasticsearch.xpack.sql.expression.RootFieldAttribute; import org.elasticsearch.xpack.sql.expression.function.Function; import org.elasticsearch.xpack.sql.expression.function.Functions; import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.sql.expression.function.aggregate.CompoundNumericAggregate; import org.elasticsearch.xpack.sql.expression.function.aggregate.Count; import org.elasticsearch.xpack.sql.expression.function.aggregate.InnerAggregate; -import org.elasticsearch.xpack.sql.expression.function.scalar.ColumnProcessor; -import org.elasticsearch.xpack.sql.expression.function.scalar.ComposeProcessor; -import org.elasticsearch.xpack.sql.expression.function.scalar.MatrixFieldProcessor; import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunctionAttribute; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.AggPathInput; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.UnresolvedInput; import org.elasticsearch.xpack.sql.plan.physical.AggregateExec; import org.elasticsearch.xpack.sql.plan.physical.EsQueryExec; import org.elasticsearch.xpack.sql.plan.physical.FilterExec; import org.elasticsearch.xpack.sql.plan.physical.LimitExec; +import org.elasticsearch.xpack.sql.plan.physical.LocalExec; import org.elasticsearch.xpack.sql.plan.physical.OrderExec; import org.elasticsearch.xpack.sql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.sql.plan.physical.ProjectExec; -import org.elasticsearch.xpack.sql.plan.physical.QuerylessExec; import org.elasticsearch.xpack.sql.planner.QueryTranslator.GroupingContext; import org.elasticsearch.xpack.sql.planner.QueryTranslator.QueryTranslation; import org.elasticsearch.xpack.sql.querydsl.agg.AggFilter; @@ -42,21 +42,23 @@ import org.elasticsearch.xpack.sql.querydsl.agg.Aggs; import org.elasticsearch.xpack.sql.querydsl.agg.GroupingAgg; import org.elasticsearch.xpack.sql.querydsl.agg.LeafAgg; import org.elasticsearch.xpack.sql.querydsl.container.AttributeSort; +import org.elasticsearch.xpack.sql.querydsl.container.ComputedRef; import org.elasticsearch.xpack.sql.querydsl.container.QueryContainer; import org.elasticsearch.xpack.sql.querydsl.container.ScriptSort; import org.elasticsearch.xpack.sql.querydsl.container.Sort.Direction; +import org.elasticsearch.xpack.sql.querydsl.container.TotalCountRef; import org.elasticsearch.xpack.sql.querydsl.query.Query; import org.elasticsearch.xpack.sql.rule.Rule; import org.elasticsearch.xpack.sql.rule.RuleExecutor; import org.elasticsearch.xpack.sql.session.EmptyExecutable; +import org.elasticsearch.xpack.sql.util.Assert; import org.elasticsearch.xpack.sql.util.StringUtils; import java.util.Arrays; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; -import static java.util.Collections.emptyMap; import static org.elasticsearch.xpack.sql.planner.QueryTranslator.and; import static org.elasticsearch.xpack.sql.planner.QueryTranslator.toAgg; import static org.elasticsearch.xpack.sql.planner.QueryTranslator.toQuery; @@ -74,15 +76,19 @@ class QueryFolder extends RuleExecutor { new FoldProject(), new FoldFilter(), new FoldOrderBy(), - new FoldLimit(), - new FoldQueryless() + new FoldLimit() + ); + + Batch local = new Batch("Local queries", + new PropagateEmptyLocal(), + new LocalLimit() ); Batch finish = new Batch("Finish query", Limiter.ONCE, new PlanOutputToQueryRef() ); - return Arrays.asList(rollup, finish); + return Arrays.asList(rollup, local, finish); } private static class FoldProject extends FoldingRule { @@ -94,53 +100,42 @@ class QueryFolder extends RuleExecutor { QueryContainer queryC = exec.queryContainer(); Map aliases = new LinkedHashMap<>(queryC.aliases()); - Map processors = new LinkedHashMap<>(queryC.processors()); + Map processors = new LinkedHashMap<>(queryC.scalarFunctions()); for (NamedExpression pj : project.projections()) { if (pj instanceof Alias) { Attribute aliasAttr = pj.toAttribute(); Expression e = ((Alias) pj).child(); - if (e instanceof ScalarFunction) { - aliases.put(aliasAttr, scalarToProcessor((ScalarFunction) e, processors)); - } - else if (e instanceof NamedExpression) { + if (e instanceof NamedExpression) { Attribute attr = ((NamedExpression) e).toAttribute(); aliases.put(aliasAttr, attr); + // add placeholder for each scalar function + if (e instanceof ScalarFunction) { + processors.put(attr, ProcessorDefinitions.toProcessorDefinition(e)); + } } else { throw new SqlIllegalArgumentException("Don't know how to translate expression %s", e); } } else { + // for named expressions nothing is recorded as these are resolved last + // otherwise 'intermediate' projects might pollute the + // output + if (pj instanceof ScalarFunction) { - aliases.put(pj.toAttribute(), scalarToProcessor((ScalarFunction) pj, processors)); + ScalarFunction f = (ScalarFunction) pj; + processors.put(f.toAttribute(), f.asProcessor()); } - // field attribute / expression } } - QueryContainer clone = new QueryContainer(queryC.query(), queryC.aggs(), queryC.refs(), aliases, processors, queryC.pseudoFunctions(), queryC.sort(), queryC.limit()); + QueryContainer clone = new QueryContainer(queryC.query(), queryC.aggs(), queryC.columns(), aliases, queryC.pseudoFunctions(), processors, queryC.sort(), queryC.limit()); return new EsQueryExec(exec.location(), exec.index(), project.output(), clone); } return project; } - - private Attribute scalarToProcessor(ScalarFunction e, Map processors) { - List trail = Functions.unwrapScalarFunctionWithTail(e); - Expression tail = trail.get(trail.size() - 1); - - ColumnProcessor proc = Functions.chainProcessors(trail); - - // in projection, scalar functions can only be applied to constants (in which case they are folded) or columns aka NamedExpressions - if (!(tail instanceof NamedExpression)) { - throw new SqlIllegalArgumentException("Expected a NamedExpression but got %s", tail); - } - - Attribute targetAttr = ((NamedExpression) tail).toAttribute(); - processors.put(targetAttr, proc); - return targetAttr; - } } private static class FoldFilter extends FoldingRule { @@ -156,7 +151,11 @@ class QueryFolder extends RuleExecutor { Query query = (qContainer.query() != null || qt.query != null) ? and(plan.location(), qContainer.query(), qt.query) : null; Aggs aggs = addPipelineAggs(qContainer, qt); - qContainer = new QueryContainer(query, aggs, qContainer.refs(), qContainer.aliases(), qContainer.processors(), qContainer.pseudoFunctions(), qContainer.sort(), qContainer.limit()); + qContainer = new QueryContainer(query, aggs, qContainer.columns(), qContainer.aliases(), + qContainer.pseudoFunctions(), + qContainer.scalarFunctions(), + qContainer.sort(), + qContainer.limit()); return exec.with(qContainer); } @@ -227,8 +226,6 @@ class QueryFolder extends RuleExecutor { // and also collect info about it (since the group columns might be used inside the select) GroupingContext groupingContext = QueryTranslator.groupBy(a.groupings()); - // shortcut used in several places - Map groupMap = groupingContext != null ? groupingContext.groupMap : emptyMap(); QueryContainer queryC = exec.queryContainer(); if (groupingContext != null) { @@ -241,85 +238,117 @@ class QueryFolder extends RuleExecutor { // followed by actual aggregates for (NamedExpression ne : a.aggregates()) { - GroupingAgg parentGroup = null; // unwrap alias - it can be // - an attribute (since we support aliases inside group-by) - // - a scalar function (used directly or acting on an argument already used for grouping) + // SELECT emp_no ... GROUP BY emp_no + // SELECT YEAR(hire_date) ... GROUP BY YEAR(hire_date) + // - an agg function (typically) + // SELECT COUNT(*), AVG(salary) ... GROUP BY salary; + + // - a scalar function, which can be applied on an attribute or aggregate and can require one or multiple inputs + + // SELECT SIN(emp_no) ... GROUP BY emp_no + // SELECT CAST(YEAR(hire_date)) ... GROUP BY YEAR(hire_date) + // SELECT CAST(AVG(salary)) ... GROUP BY salary + // SELECT AVG(salary) + SIN(MIN(salary)) ... GROUP BY salary if (ne instanceof Alias || ne instanceof Function) { Alias as = ne instanceof Alias ? (Alias) ne : null; Expression child = as != null ? as.child() : ne; // record aliases in case they are later referred in the tree - if (as != null) { + if (as != null && as.child() instanceof NamedExpression) { aliases.put(as.toAttribute(), ((NamedExpression) as.child()).toAttribute()); } // // look first for scalar functions which might wrap the actual grouped target - // (e.g. CAST(field) or ABS(field), etc.. - // + // (e.g. + // CAST(field) GROUP BY field or + // ABS(YEAR(field)) GROUP BY YEAR(field) or + // ABS(AVG(salary)) ... GROUP BY salary) + // ) + if (child instanceof ScalarFunction) { + ScalarFunction f = (ScalarFunction) child; + ProcessorDefinition proc = f.asProcessor(); - List wrappingFunctions = Functions.unwrapScalarFunctionWithTail(child); - ColumnProcessor proc = null; - Expression resolvedGroupedExp = null; - int resolvedExpIndex = -1; + final AtomicReference qC = new AtomicReference<>(queryC); - // look-up the hierarchy to match the group - for (int i = wrappingFunctions.size() - 1; i >= 0 && resolvedGroupedExp == null; i--) { - Expression exp = wrappingFunctions.get(i); - parentGroup = groupingContext != null ? groupingContext.parentGroupFor(exp) : null; + proc = proc.transformUp(p -> { + // get the backing expression and check if it belongs to a agg group or whether it's an expression in the first place + Expression exp = p.expression(); + GroupingAgg matchingGroup = null; + if (groupingContext != null) { + // is there a group (aggregation) for this expression ? + matchingGroup = groupingContext.groupFor(exp); + } + else { + // a scalar function can be used only if has already been mentioned for grouping + // (otherwise it is the opposite of grouping) + if (exp instanceof ScalarFunction) { + throw new SqlIllegalArgumentException("Scalar function %s can be used only if included already in grouping", exp.toString()); + } + } - // found group for expression or bumped into an aggregate (can happen when dealing with a root group) - if (parentGroup != null || Functions.isAggregateFunction(exp)) { - resolvedGroupedExp = exp; - resolvedExpIndex = i; - } - } - // if needed, combine the wrapping functions as processors - if (resolvedExpIndex >= 0) { - // sublist has the upper index exclusive hence the +1 - proc = Functions.chainProcessors(wrappingFunctions.subList(0, resolvedExpIndex + 1)); - } - - // if there was no unwrapping, fallback to child - if (resolvedGroupedExp == null) { - resolvedGroupedExp = child; - } - - // initialize parent if needed - parentGroup = parentGroup == null && groupingContext != null ? groupingContext.parentGroupFor(resolvedGroupedExp) : parentGroup; - - if (resolvedGroupedExp instanceof Attribute) { - queryC = useNamedReference(((Attribute) resolvedGroupedExp), proc, groupMap, queryC); - } - - // a scalar function can be used only if has been already used for grouping - // otherwise it is the opposite of grouping - else if (Functions.isScalarFunction(resolvedGroupedExp)) { - ScalarFunction sf = (ScalarFunction) resolvedGroupedExp; + // found match for expression; if it's an attribute or scalar, end the processing chain with the reference to the backing agg + if (matchingGroup != null) { + if (exp instanceof Attribute || exp instanceof ScalarFunction) { + return new AggPathInput(exp, matchingGroup.propertyPath()); + } + } + // or found an aggregate expression (which has to work on an attribute used for grouping) + // (can happen when dealing with a root group) + if (Functions.isAggregateFunction(exp)) { + Tuple withFunction = addAggFunction(matchingGroup, (AggregateFunction) exp, compoundAggMap, qC.get()); + qC.set(withFunction.v1()); + return withFunction.v2(); + } + // not an aggregate and no matching group means trouble + throw new SqlIllegalArgumentException("Cannot find group '%s'", Expressions.name(exp)); + }, UnresolvedInput.class); - if (parentGroup == null) { - throw new SqlIllegalArgumentException("Scalar function %s can be used only if included already in grouping", sf.name()); - } - - queryC = queryC.addAggRef(parentGroup.propertyPath(), proc); - // redirect the alias to the scalar group id (changing the id altogether doesn't work since the agg path already uses it) - aliases.put(as.toAttribute(), sf.toAttribute().withId(sf.id())); + // add the computed column + queryC = qC.get().addColumn(new ComputedRef(proc)); + + // TODO: is this needed? + // redirect the alias to the scalar group id (changing the id altogether doesn't work it is already used in the aggpath) + //aliases.put(as.toAttribute(), sf.toAttribute()); } + // apply the same logic above (for function inputs) to non-scalar functions with small variantions: + // instead of adding things as input, add them as full blown column else { - if (!Functions.isAggregateFunction(resolvedGroupedExp)) { - throw new SqlIllegalArgumentException("Expected aggregate function inside alias; got %s", child.nodeString()); + GroupingAgg matchingGroup = null; + if (groupingContext != null) { + // is there a group (aggregation) for this expression ? + matchingGroup = groupingContext.groupFor(child); + } + // attributes can only refer to declared groups + if (child instanceof Attribute) { + Assert.notNull(matchingGroup, "Cannot find group '%s'", Expressions.name(child)); + queryC = queryC.addAggColumn(matchingGroup.propertyPath()); + } + else { + // the only thing left is agg function + Assert.isTrue(Functions.isAggregateFunction(child), "Expected aggregate function inside alias; got %s", child.nodeString()); + Tuple withAgg = addAggFunction(matchingGroup, (AggregateFunction) child, compoundAggMap, queryC); + //FIXME: what about inner key + queryC = withAgg.v1().addAggColumn(withAgg.v2().context()); + if (withAgg.v2().innerKey() != null) { + throw new UnsupportedOperationException("innerkey/matrix stats not handled"); + } } - AggregateFunction f = (AggregateFunction) resolvedGroupedExp; - queryC = addFunction(parentGroup, f, proc, compoundAggMap, queryC); } } - // not an Alias, means it's an Attribute + // not an Alias or a Function, means it's an Attribute so apply the same logic as above else { - queryC = useNamedReference(ne, null, groupMap, queryC); + GroupingAgg matchingGroup = null; + if (groupingContext != null) { + matchingGroup = groupingContext.groupFor(ne); + Assert.notNull(matchingGroup, "Cannot find group '%s'", Expressions.name(ne)); + queryC = queryC.addAggColumn(matchingGroup.propertyPath()); + } } } @@ -333,27 +362,22 @@ class QueryFolder extends RuleExecutor { return a; } - // the agg is an actual value (field) that points to a group - // so look it up and create an extractor for it - private QueryContainer useNamedReference(NamedExpression ne, ColumnProcessor proc, Map groupMap, QueryContainer queryC) { - GroupingAgg aggInfo = groupMap.get(ne.id()); - if (aggInfo == null) { - throw new SqlIllegalArgumentException("Cannot find group '%s'", ne.name()); - } - return queryC.addAggRef(aggInfo.propertyPath(), proc); - } - - private QueryContainer addFunction(GroupingAgg parentAgg, AggregateFunction f, ColumnProcessor proc, Map compoundAggMap, QueryContainer queryC) { + private Tuple addAggFunction(GroupingAgg parentAgg, AggregateFunction f, Map compoundAggMap, QueryContainer queryC) { String functionId = f.functionId(); // handle count as a special case agg if (f instanceof Count) { Count c = (Count) f; if (!c.distinct()) { - return queryC.addAggCount(parentAgg, functionId, proc); + String path = parentAgg == null ? TotalCountRef.PATH : AggPath.bucketCount(parentAgg.asParentPath()); + Map pseudoFunctions = new LinkedHashMap<>(queryC.pseudoFunctions()); + pseudoFunctions.put(functionId, parentAgg); + return new Tuple<>(queryC.withPseudoFunctions(pseudoFunctions), new AggPathInput(f, path)); } } - // otherwise translate it to an agg + AggPathInput aggInput = null; + + // otherwise translate the function into an actual agg String parentPath = parentAgg != null ? parentAgg.asParentPath() : null; String groupId = parentAgg != null ? parentAgg.id() : null; @@ -367,20 +391,22 @@ class QueryFolder extends RuleExecutor { LeafAgg leafAgg = toAgg(parentPath, functionId, outer); cAggPath = leafAgg.propertyPath(); compoundAggMap.put(outer, cAggPath); - // add the agg without the default ref to it + // add the agg (without any reference) queryC = queryC.with(queryC.aggs().addAgg(leafAgg)); } - + String aggPath = AggPath.metricValue(cAggPath, ia.innerId()); - // FIXME: concern leak - hack around MatrixAgg which is not generalized (afaik) - if (ia.innerKey() != null) { - proc = new ComposeProcessor(new MatrixFieldProcessor(QueryTranslator.nameOf(ia.innerKey())), proc); - } - - return queryC.addAggRef(aggPath, proc); + // FIXME: concern leak - hack around MatrixAgg which is not + // generalized (afaik) + aggInput = new AggPathInput(f, aggPath, ia.innerKey() != null ? QueryTranslator.nameOf(ia.innerKey()) : null); } - - return queryC.addAgg(groupId, toAgg(parentPath, functionId, f), proc); + else { + LeafAgg leafAgg = toAgg(parentPath, functionId, f); + aggInput = new AggPathInput(f, leafAgg.propertyPath()); + queryC = queryC.with(queryC.aggs().addAgg(groupId, leafAgg)); + } + + return new Tuple<>(queryC, aggInput); } } @@ -463,47 +489,55 @@ class QueryFolder extends RuleExecutor { } } - private static class FoldQueryless extends FoldingRule { - - @Override - protected PhysicalPlan rule(PhysicalPlan plan) { - if (plan.children().size() == 1 && plan.children().get(0) instanceof QuerylessExec) { - return new QuerylessExec(plan.location(), new EmptyExecutable(plan.output())); - } - return plan; - } - } - private static class PlanOutputToQueryRef extends FoldingRule { @Override protected PhysicalPlan rule(EsQueryExec exec) { QueryContainer qContainer = exec.queryContainer(); // references (aka aggs) are in place - if (qContainer.hasReferences()) { + if (qContainer.hasColumns()) { return exec; } for (Attribute attr : exec.output()) { - if (attr instanceof ScalarFunctionAttribute) { - attr = qContainer.aliases().get(attr); - } - if (attr instanceof RootFieldAttribute) { - qContainer = qContainer.addFieldRef((RootFieldAttribute) attr); - } - else if (attr instanceof NestedFieldAttribute) { - NestedFieldAttribute nfa = (NestedFieldAttribute) attr; - qContainer = qContainer.addNestedFieldRef(nfa); - } - else { - throw new SqlIllegalArgumentException("Unknown output attribute %s", attr); - } + qContainer = qContainer.addColumn(attr); } + // after all attributes have been resolved return exec.with(qContainer); } } + // + // local + // + + private static class PropagateEmptyLocal extends FoldingRule { + + @Override + protected PhysicalPlan rule(PhysicalPlan plan) { + if (plan.children().size() == 1) { + PhysicalPlan p = plan.children().get(0); + if (p instanceof LocalExec && ((LocalExec) p).isEmpty()) { + return new LocalExec(plan.location(), new EmptyExecutable(plan.output())); + } + } + return plan; + } + } + + // local exec currently means empty or one entry so limit can't really be applied + private static class LocalLimit extends FoldingRule { + + @Override + protected PhysicalPlan rule(LimitExec plan) { + if (plan.child() instanceof LocalExec) { + return plan.child(); + } + return plan; + } + } + // rule for folding physical plans together abstract static class FoldingRule extends Rule { diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java index e11389538cf..1e714592af4 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java @@ -208,7 +208,7 @@ abstract class QueryTranslator { } - GroupingAgg parentGroupFor(Expression exp) { + GroupingAgg groupFor(Expression exp) { if (Functions.isAggregateFunction(exp)) { AggregateFunction f = (AggregateFunction) exp; // if there's at least one agg in the tree @@ -372,7 +372,7 @@ abstract class QueryTranslator { static String nameOf(Expression e) { if (e instanceof DateTimeFunction) { - return nameOf(((DateTimeFunction) e).argument()); + return nameOf(((DateTimeFunction) e).field()); } if (e instanceof NamedExpression) { return ((NamedExpression) e).name(); @@ -516,7 +516,7 @@ abstract class QueryTranslator { @Override protected QueryTranslation asQuery(BinaryComparison bc, boolean onAggs) { - Assert.isTrue(bc.right() instanceof Literal, "don't know how to translate right %s in %s", bc.right().nodeString(), bc); + Assert.isTrue(bc.right().foldable(), "don't know how to translate right %s in %s", bc.right().nodeString(), bc); if (bc.left() instanceof NamedExpression) { NamedExpression ne = (NamedExpression) bc.left(); diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/AggRef.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/AggRef.java index 12a31309299..9b6fed48522 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/AggRef.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/AggRef.java @@ -7,11 +7,11 @@ package org.elasticsearch.xpack.sql.querydsl.container; import org.elasticsearch.xpack.sql.querydsl.agg.AggPath; -public class AggRef implements Reference { +public class AggRef implements ColumnReference { private final String path; private final int depth; - AggRef(String path) { + public AggRef(String path) { this.path = path; depth = AggPath.depth(path); } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/ColumnReference.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/ColumnReference.java new file mode 100644 index 00000000000..17a8c551c7f --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/ColumnReference.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.querydsl.container; + +/** + * Entity representing a 'column' backed by one or multiple results from ES. + */ +public interface ColumnReference { + + /** + * Indicates the depth of the result. Used for counting the actual size of a + * result by knowing how many nested levels there are. Typically used by + * aggregations. + * + * @return depth of the result + */ + int depth(); +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/ComputedRef.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/ComputedRef.java new file mode 100644 index 00000000000..de152e8ea95 --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/ComputedRef.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.querydsl.container; + +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ReferenceInput; + +import java.util.concurrent.atomic.AtomicInteger; + +public class ComputedRef implements ColumnReference { + + private final ProcessorDefinition processor; + private final int depth; + + public ComputedRef(ProcessorDefinition processor) { + this.processor = processor; + + // compute maximum depth + AtomicInteger d = new AtomicInteger(0); + processor.forEachDown(i -> { + ColumnReference ref = i.context(); + if (ref.depth() > d.get()) { + d.set(ref.depth()); + } + }, ReferenceInput.class); + + depth = d.get(); + } + + public ProcessorDefinition processor() { + return processor; + } + + @Override + public int depth() { + return depth; + } + + @Override + public String toString() { + return processor + "(" + processor + ")"; + } +} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/FieldReference.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/FieldReference.java index 7db36705ae0..b3d411749ff 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/FieldReference.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/FieldReference.java @@ -5,7 +5,17 @@ */ package org.elasticsearch.xpack.sql.querydsl.container; -public interface FieldReference extends Reference { +public interface FieldReference extends ColumnReference { + @Override + default int depth() { + return 0; + } + + /** + * Field name. + * + * @return field name. + */ String name(); } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/ProcessingRef.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/ProcessingRef.java deleted file mode 100644 index 18915c1e9f1..00000000000 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/ProcessingRef.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.sql.querydsl.container; - -import org.elasticsearch.xpack.sql.expression.function.scalar.ColumnProcessor; - -public class ProcessingRef implements Reference { - - private final ColumnProcessor processor; - private final Reference ref; - - public ProcessingRef(ColumnProcessor processor, Reference ref) { - this.processor = processor; - this.ref = ref; - } - - public ColumnProcessor processor() { - return processor; - } - - public Reference ref() { - return ref; - } - - @Override - public String toString() { - return processor + "(" + ref + ")"; - } -} diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java index 97b9040e55c..bdf846033f9 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java @@ -5,15 +5,16 @@ */ package org.elasticsearch.xpack.sql.querydsl.container; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.execution.search.SourceGenerator; import org.elasticsearch.xpack.sql.expression.Attribute; -import org.elasticsearch.xpack.sql.expression.FieldAttribute; import org.elasticsearch.xpack.sql.expression.NestedFieldAttribute; import org.elasticsearch.xpack.sql.expression.RootFieldAttribute; -import org.elasticsearch.xpack.sql.expression.function.scalar.ColumnProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunctionAttribute; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.AttributeInput; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ReferenceInput; import org.elasticsearch.xpack.sql.querydsl.agg.AggPath; import org.elasticsearch.xpack.sql.querydsl.agg.Aggs; import org.elasticsearch.xpack.sql.querydsl.agg.GroupingAgg; @@ -22,8 +23,8 @@ import org.elasticsearch.xpack.sql.querydsl.query.AndQuery; import org.elasticsearch.xpack.sql.querydsl.query.MatchAll; import org.elasticsearch.xpack.sql.querydsl.query.NestedQuery; import org.elasticsearch.xpack.sql.querydsl.query.Query; +import org.elasticsearch.xpack.sql.util.StringUtils; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashMap; @@ -33,6 +34,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; @@ -44,14 +46,21 @@ public class QueryContainer { private final Aggs aggs; private final Query query; - private final List refs; + + // final output seen by the client (hence the list or ordering) + // gets converted by the Scroller into Extractors for hits or actual results in case of aggregations + private final List columns; + // aliases (maps an alias to its actual resolved attribute) private final Map aliases; - // processors for a given attribute - wraps the processor over the resolved ref - private final Map processors; + // pseudo functions (like count) - that are 'extracted' from other aggs private final Map pseudoFunctions; + // scalar function processors - recorded as functions get folded; + // at scrolling, their inputs (leaves) get updated + private final Map scalarFunctions; + private final Set sort; private final int limit; @@ -63,26 +72,31 @@ public class QueryContainer { this(null, null, null, null, null, null, null, -1); } - public QueryContainer(Query query, Aggs aggs, List refs, Map aliases, Map processors, Map pseudoFunctions, Set sort, int limit) { + public QueryContainer(Query query, Aggs aggs, List refs, Map aliases, + Map pseudoFunctions, + Map scalarFunctions, + Set sort, int limit) { this.query = query; this.aggs = aggs == null ? new Aggs() : aggs; this.aliases = aliases == null || aliases.isEmpty() ? emptyMap() : aliases; - this.processors = processors == null || processors.isEmpty() ? emptyMap() : processors; this.pseudoFunctions = pseudoFunctions == null || pseudoFunctions.isEmpty() ? emptyMap() : pseudoFunctions; - this.refs = refs == null || refs.isEmpty() ? emptyList() : refs; + this.scalarFunctions = scalarFunctions == null || scalarFunctions.isEmpty() ? emptyMap() : scalarFunctions; + this.columns = refs == null || refs.isEmpty() ? emptyList() : refs; this.sort = sort == null || sort.isEmpty() ? emptySet() : sort; this.limit = limit; int aggLevel = 0; boolean onlyAggs = true; - for (Reference ref : this.refs) { + for (ColumnReference ref : this.columns) { if (ref.depth() > aggLevel) { aggLevel = ref.depth(); } - while (ref instanceof ProcessingRef) { - ProcessingRef r = (ProcessingRef) ref; - ref = r.ref(); + if (ref instanceof ComputedRef) { + // check field references + if (((ComputedRef) ref).processor().anyMatch(p -> p instanceof ReferenceInput && ((ReferenceInput) p).context() instanceof FieldReference)) { + onlyAggs = false; + } } if (ref instanceof FieldReference) { onlyAggs = false; @@ -100,18 +114,14 @@ public class QueryContainer { return aggs; } - public List refs() { - return refs; + public List columns() { + return columns; } public Map aliases() { return aliases; } - public Map processors() { - return processors; - } - public Map pseudoFunctions() { return pseudoFunctions; } @@ -132,8 +142,8 @@ public class QueryContainer { return aggDepth; } - public boolean hasReferences() { - return !refs.isEmpty(); + public boolean hasColumns() { + return !columns.isEmpty(); } // @@ -141,72 +151,59 @@ public class QueryContainer { // public QueryContainer with(Query q) { - return new QueryContainer(q, aggs, refs, aliases, processors, pseudoFunctions, sort, limit); + return new QueryContainer(q, aggs, columns, aliases, pseudoFunctions, scalarFunctions, sort, limit); } - public QueryContainer with(List r) { - return new QueryContainer(query, aggs, r, aliases, processors, pseudoFunctions, sort, limit); + public QueryContainer with(List r) { + return new QueryContainer(query, aggs, r, aliases, pseudoFunctions, scalarFunctions, sort, limit); } public QueryContainer withAliases(Map a) { - return new QueryContainer(query, aggs, refs, a, processors, pseudoFunctions, sort, limit); - } - - public QueryContainer withProcessors(Map p) { - return new QueryContainer(query, aggs, refs, aliases, p, pseudoFunctions, sort, limit); + return new QueryContainer(query, aggs, columns, a, pseudoFunctions, scalarFunctions, sort, limit); } public QueryContainer withPseudoFunctions(Map p) { - return new QueryContainer(query, aggs, refs, aliases, processors, p, sort, limit); + return new QueryContainer(query, aggs, columns, aliases, p, scalarFunctions, sort, limit); } public QueryContainer with(Aggs a) { - return new QueryContainer(query, a, refs, aliases, processors, pseudoFunctions, sort, limit); + return new QueryContainer(query, a, columns, aliases, pseudoFunctions, scalarFunctions, sort, limit); } public QueryContainer withLimit(int l) { - return l == limit ? this : new QueryContainer(query, aggs, refs, aliases, processors, pseudoFunctions, sort, l); + return l == limit ? this : new QueryContainer(query, aggs, columns, aliases, pseudoFunctions, scalarFunctions, sort, l); } + public QueryContainer withScalarProcessors(Map procs) { + return new QueryContainer(query, aggs, columns, aliases, pseudoFunctions, procs, sort, limit); + } public QueryContainer sort(Sort sortable) { Set sort = new LinkedHashSet<>(this.sort); sort.add(sortable); - return new QueryContainer(query, aggs, refs, aliases, processors, pseudoFunctions, sort, limit); + return new QueryContainer(query, aggs, columns, aliases, pseudoFunctions, scalarFunctions, sort, limit); } private String aliasName(Attribute attr) { return aliases.getOrDefault(attr, attr).name(); } - private Reference wrapProcessorIfNeeded(Attribute attr, Reference ref) { - ColumnProcessor columnProcessor = processors.get(attr); - return columnProcessor != null ? new ProcessingRef(columnProcessor, ref) : ref; - } - // // reference methods // - public QueryContainer addFieldRef(RootFieldAttribute fieldAttr) { - SearchHitFieldRef fieldRef = new SearchHitFieldRef(aliasName(fieldAttr), shouldUseDocValue(fieldAttr)); - return addRef(wrapProcessorIfNeeded(fieldAttr, fieldRef)); + private ColumnReference fieldRef(RootFieldAttribute fieldAttr) { + return new SearchHitFieldRef(aliasName(fieldAttr), fieldAttr.dataType().hasDocValues()); } - private boolean shouldUseDocValue(FieldAttribute fieldAttr) { - // TODO: configurable retrieval format for dates - // && !(fieldAttr.dataType() instanceof DateType) - return fieldAttr.dataType().hasDocValues(); - } - - public QueryContainer addNestedFieldRef(NestedFieldAttribute attr) { + private Tuple nestedFieldRef(NestedFieldAttribute attr) { // attach the field to the relevant nested query - List nestedRefs = new ArrayList<>(); + List nestedRefs = new ArrayList<>(); String parent = attr.parentPath(); String name = aliasName(attr); Query q = query; - Map field = singletonMap(name, shouldUseDocValue(attr)); + Map field = singletonMap(name, attr.dataType().hasDocValues()); if (q == null) { q = new NestedQuery(attr.location(), parent, field, new MatchAll(attr.location())); } @@ -231,49 +228,89 @@ public class QueryContainer { } } - NestedFieldRef nestedFieldRef = new NestedFieldRef(attr.parentPath(), attr.name(), shouldUseDocValue(attr)); + NestedFieldRef nestedFieldRef = new NestedFieldRef(attr.parentPath(), attr.name(), attr.dataType().hasDocValues()); + nestedRefs.add(nestedFieldRef); - nestedRefs.add(wrapProcessorIfNeeded(attr, nestedFieldRef)); - - return new QueryContainer(q, aggs, combine(refs, nestedRefs), aliases, processors, pseudoFunctions, sort, limit); + return new Tuple<>(new QueryContainer(q, aggs, columns, aliases, pseudoFunctions, scalarFunctions, sort, limit), nestedFieldRef); } - private QueryContainer addRef(Reference ref) { - return with(combine(refs, ref)); + // replace function's input with references + private Tuple computingRef(ScalarFunctionAttribute sfa) { + Attribute name = aliases.getOrDefault(sfa, sfa); + ProcessorDefinition proc = scalarFunctions.get(name); + + // check the attribute itself + if (proc == null) { + if (name instanceof ScalarFunctionAttribute) { + sfa = (ScalarFunctionAttribute) name; + } + proc = sfa.processorDef(); + } + AtomicReference containerRef = new AtomicReference(this); + + // find the processor inputs (Attributes) and convert them into references + // no need to promote them to the top since the container doesn't have to be aware + proc = proc.transformUp(l -> { + Attribute attr = aliases.getOrDefault(l.context(), l.context()); + Tuple ref = containerRef.get().toReference(attr); + containerRef.set(ref.v1()); + return new ReferenceInput(l.expression(), ref.v2()); + }, AttributeInput.class); + + QueryContainer qContainer = containerRef.get(); + // update proc + Map procs = new LinkedHashMap<>(qContainer.scalarFunctions()); + procs.put(name, proc); + qContainer = qContainer.withScalarProcessors(procs); + return new Tuple<>(qContainer, new ComputedRef(proc)); + } + + public QueryContainer addColumn(Attribute attr) { + Tuple tuple = toReference(attr); + return tuple.v1().addColumn(tuple.v2()); + } + + private Tuple toReference(Attribute attr) { + if (attr instanceof RootFieldAttribute) { + return new Tuple<>(this, fieldRef((RootFieldAttribute) attr)); + } + if (attr instanceof NestedFieldAttribute) { + return nestedFieldRef((NestedFieldAttribute) attr); + } + if (attr instanceof ScalarFunctionAttribute) { + return computingRef((ScalarFunctionAttribute) attr); + } + throw new SqlIllegalArgumentException("Unknown output attribute %s", attr); + } + + public QueryContainer addColumn(ColumnReference ref) { + return with(combine(columns, ref)); + } + + public Map scalarFunctions() { + return scalarFunctions; } // // agg methods // - public QueryContainer addAggRef(String aggPath) { - return addAggRef(aggPath, null); + public QueryContainer addAggColumn(String aggPath) { + return with(combine(columns, new AggRef(aggPath))); } - public QueryContainer addAggRef(String aggPath, ColumnProcessor processor) { - return addAggRef(new AggRef(aggPath), processor); - } - - public QueryContainer addAggRef(AggRef customRef, ColumnProcessor processor) { - Reference ref = processor != null ? new ProcessingRef(processor, customRef) : customRef; - return addRef(ref); - } - - public QueryContainer addAggCount(GroupingAgg parentGroup, String functionId, ColumnProcessor processor) { - Reference ref = parentGroup == null ? TotalCountRef.INSTANCE : new AggRef(AggPath.bucketCount(parentGroup.asParentPath())); - ref = processor != null ? new ProcessingRef(processor, ref) : ref; + public QueryContainer addAggCount(GroupingAgg parentGroup, String functionId) { + ColumnReference ref = parentGroup == null ? TotalCountRef.INSTANCE : new AggRef(AggPath.bucketCount(parentGroup.asParentPath())); Map pseudoFunctions = new LinkedHashMap<>(this.pseudoFunctions); pseudoFunctions.put(functionId, parentGroup); - return new QueryContainer(query, aggs, combine(refs, ref), aliases, processors, pseudoFunctions, sort, limit); + return new QueryContainer(query, aggs, combine(columns, ref), aliases, pseudoFunctions, scalarFunctions, sort, limit); } - public QueryContainer addAgg(String groupId, LeafAgg agg, ColumnProcessor processor) { - return addAgg(groupId, agg, agg.propertyPath(), processor); + public QueryContainer addAgg(String groupId, LeafAgg agg) { + return addAgg(groupId, agg, agg.propertyPath()); } - public QueryContainer addAgg(String groupId, LeafAgg agg, String aggRefPath, ColumnProcessor processor) { - AggRef aggRef = new AggRef(aggRefPath); - Reference ref = processor != null ? new ProcessingRef(processor, aggRef) : aggRef; - return new QueryContainer(query, aggs.addAgg(groupId, agg), combine(refs, ref), aliases, processors, pseudoFunctions, sort, limit); + public QueryContainer addAgg(String groupId, LeafAgg agg, String aggRefPath) { + return new QueryContainer(query, aggs.addAgg(groupId, agg), columns, aliases, pseudoFunctions, scalarFunctions, sort, limit); } public QueryContainer addGroups(Collection values) { @@ -294,7 +331,7 @@ public class QueryContainer { @Override public int hashCode() { - return Objects.hash(query, aggs, refs, aliases); + return Objects.hash(query, aggs, columns, aliases); } @Override @@ -310,7 +347,7 @@ public class QueryContainer { QueryContainer other = (QueryContainer) obj; return Objects.equals(query, other.query) && Objects.equals(aggs, other.aggs) - && Objects.equals(refs, other.refs) + && Objects.equals(columns, other.columns) && Objects.equals(aliases, other.aliases) && Objects.equals(sort, other.sort) && Objects.equals(limit, other.limit); @@ -318,12 +355,7 @@ public class QueryContainer { @Override public String toString() { - try (XContentBuilder builder = JsonXContent.contentBuilder()) { - builder.humanReadable(true).prettyPrint(); - SourceGenerator.sourceBuilder(this).toXContent(builder, ToXContent.EMPTY_PARAMS); - return builder.string(); - } catch (IOException e) { - throw new RuntimeException("error rendering", e); - } + // annoying cycle (QC depends on SC which depends on QC) + return StringUtils.toString(SourceGenerator.sourceBuilder(this)); } -} +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/TotalCountRef.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/TotalCountRef.java index 9cc23b596cb..8ce817c4943 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/TotalCountRef.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/TotalCountRef.java @@ -9,7 +9,9 @@ import org.elasticsearch.xpack.sql.util.StringUtils; // somewhat of a fake agg (since it gets optimized and it gets its value from the response) public final class TotalCountRef extends AggRef { - public static TotalCountRef INSTANCE = new TotalCountRef(); + public static final TotalCountRef INSTANCE = new TotalCountRef(); + + public static final String PATH = "#_count_#"; TotalCountRef() { super(StringUtils.EMPTY); diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/session/Cursor.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/session/Cursor.java index de11befbbd4..e638c176f17 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/session/Cursor.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/session/Cursor.java @@ -10,8 +10,8 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.io.FastStringReader; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.xpack.sql.execution.search.HitExtractor; import org.elasticsearch.xpack.sql.execution.search.ScrollCursor; +import org.elasticsearch.xpack.sql.execution.search.extractor.HitExtractors; import java.io.IOException; import java.io.StringWriter; @@ -38,7 +38,7 @@ public interface Cursor extends NamedWriteable { */ static List getNamedWriteables() { List entries = new ArrayList<>(); - entries.addAll(HitExtractor.getNamedWriteables()); + entries.addAll(HitExtractors.getNamedWriteables()); entries.add(new NamedWriteableRegistry.Entry(Cursor.class, EmptyCursor.NAME, in -> EMPTY)); entries.add(new NamedWriteableRegistry.Entry(Cursor.class, ScrollCursor.NAME, ScrollCursor::new)); return entries; diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/session/SingletonExecutable.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/session/SingletonExecutable.java new file mode 100644 index 00000000000..a81a7bb33db --- /dev/null +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/session/SingletonExecutable.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.session; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.xpack.sql.expression.Attribute; +import org.elasticsearch.xpack.sql.util.Assert; + +import java.util.List; + +public class SingletonExecutable implements Executable { + + private final List output; + private final Object[] values; + + public SingletonExecutable(List output, Object... values) { + Assert.isTrue(output.size() == values.length, "Output %s and values %s are out of sync", output, values); + this.output = output; + this.values = values; + } + + @Override + public List output() { + return output; + } + + @Override + public void execute(SqlSession session, ActionListener listener) { + listener.onResponse(Rows.singleton(output, values)); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < values.length; i++) { + sb.append(output.get(i)); + sb.append("="); + sb.append(values[i]); + } + return sb.toString(); + } +} \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/session/SqlSettings.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/session/SqlSettings.java index 82005e1c3f0..f7ec0f96ebf 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/session/SqlSettings.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/session/SqlSettings.java @@ -8,6 +8,8 @@ package org.elasticsearch.xpack.sql.session; import org.elasticsearch.common.settings.Settings; import org.joda.time.DateTimeZone; +import java.util.TimeZone; + // Typed object holding properties for a given public class SqlSettings { @@ -40,7 +42,8 @@ public class SqlSettings { } public DateTimeZone timeZone() { - return DateTimeZone.forID(timeZoneId()); + // use this instead of DateTimeZone#forID because DTZ doesn't support all of j.u.TZ IDs (such as IST) + return DateTimeZone.forTimeZone(TimeZone.getTimeZone(TIMEZONE_ID)); } public int pageSize() { diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/tree/NodeUtils.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/tree/NodeUtils.java index db1ebeb3aa7..c52521470ca 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/tree/NodeUtils.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/tree/NodeUtils.java @@ -68,7 +68,7 @@ public abstract class NodeUtils { // public Literal left() { return left; } // public Literal right() { return right; } // } - static > T copyTree(Node tree, List newChildren) { + public static > T copyTree(Node tree, List newChildren) { Assert.notNull(tree, "Non-null tree expected"); // basic sanity check @@ -164,7 +164,8 @@ public abstract class NodeUtils { // validate return type Class expected = param.getType(); Class found = getter.getReturnType(); - Assert.isTrue(expected.isAssignableFrom(found), "Constructor param [%s] in class [%s] has type [%s] but found getter [%s]", paramName, clazz, expected, getter.toGenericString()); + // found == Object if we're dealing with generics + Assert.isTrue(found == Object.class || expected.isAssignableFrom(found), "Constructor param [%s] in class [%s] has type [%s] but found getter [%s]", paramName, clazz, expected, getter.toGenericString()); params.put(paramName, getter); } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java index 42f34b6847f..8e10d97e573 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java @@ -21,7 +21,48 @@ public abstract class DataTypeConversion { private static final DateTimeFormatter UTC_DATE_FORMATTER = ISODateTimeFormat.dateTimeNoMillis().withZoneUTC(); - public static boolean nullable(DataType from, DataType to) { + public static DataType commonType(DataType left, DataType right) { + if (left.same(right)) { + return left; + } + if (nullable(left)) { + return right; + } + if (nullable(right)) { + return left; + } + if (left.isNumeric() && right.isNumeric()) { + // if one is int + if (left.isInteger()) { + // promote the highest int + if (right.isInteger()) { + return left.size() > right.size() ? left : right; + } + // promote the rational + return right; + } + // try the other side + if (right.isInteger()) { + return left; + } + // promote the highest rational + return left.size() > right.size() ? left : right; + } + if (left instanceof StringType) { + if (right.isNumeric()) { + return right; + } + } + if (right instanceof StringType) { + if (left.isNumeric()) { + return left; + } + } + // none found + return null; + } + + public static boolean nullable(DataType from) { return from instanceof NullType; } @@ -222,21 +263,21 @@ public abstract class DataTypeConversion { throw new SqlIllegalArgumentException("cannot convert [" + from + "] to [Boolean]"); } - private static byte safeToByte(long x) { + public static byte safeToByte(long x) { if (x > Byte.MAX_VALUE || x < Byte.MIN_VALUE) { throw new SqlIllegalArgumentException("Numeric %d out of byte range", Long.toString(x)); } return (byte) x; } - private static short safeToShort(long x) { + public static short safeToShort(long x) { if (x > Short.MAX_VALUE || x < Short.MIN_VALUE) { throw new SqlIllegalArgumentException("Numeric %d out of short range", Long.toString(x)); } return (short) x; } - private static int safeToInt(long x) { + public static int safeToInt(long x) { if (x > Integer.MAX_VALUE || x < Integer.MIN_VALUE) { // NOCOMMIT should these instead be regular IllegalArgumentExceptions so we throw a 400 error? Or something else? throw new SqlIllegalArgumentException("numeric %d out of int range", Long.toString(x)); @@ -244,13 +285,21 @@ public abstract class DataTypeConversion { return (int) x; } - private static long safeToLong(double x) { + public static long safeToLong(double x) { if (x > Long.MAX_VALUE || x < Long.MIN_VALUE) { throw new SqlIllegalArgumentException("[" + x + "] out of [Long] range"); } return Math.round(x); } + public static Object convert(Object value, DataType dataType) { + DataType detectedType = DataTypes.fromJava(value); + if (detectedType.equals(dataType)) { + return value; + } + return conversionFor(detectedType, dataType).convert(value); + } + /** * Reference to a data type conversion that can be serialized. Note that the position in the enum * is important because it is used for serialization. @@ -306,9 +355,9 @@ public abstract class DataTypeConversion { try { return converter.apply(value.toString()); } catch (NumberFormatException e) { - throw new SqlIllegalArgumentException("cannot cast [" + value + "] to [" + to + "]", e); + throw new SqlIllegalArgumentException("cannot cast [%s] to [%s]", value, to, e); } catch (IllegalArgumentException e) { - throw new SqlIllegalArgumentException("cannot cast [" + value + "] to [" + to + "]: " + e.getMessage(), e); + throw new SqlIllegalArgumentException("cannot cast [%s] to [%s]:%s", value, to, e.getMessage(), e); } }; } @@ -324,4 +373,12 @@ public abstract class DataTypeConversion { return converter.apply(l); } } + + public static DataType asInteger(DataType dataType) { + if (!dataType.isNumeric()) { + return dataType; + } + + return dataType.isInteger() ? dataType : DataTypes.LONG; + } } \ No newline at end of file diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/type/DataTypes.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/type/DataTypes.java index d458152e57c..9dc894e47db 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/type/DataTypes.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/type/DataTypes.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.sql.type; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; +import org.joda.time.DateTime; import java.sql.JDBCType; import java.util.LinkedHashMap; @@ -104,6 +105,9 @@ public abstract class DataTypes { if (value instanceof Short) { return SHORT; } + if (value instanceof DateTime) { + return DATE; + } if (value instanceof String) { return KEYWORD; } diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/util/CollectionUtils.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/util/CollectionUtils.java index d0da23cf4e0..78704c5e1ca 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/util/CollectionUtils.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/util/CollectionUtils.java @@ -8,13 +8,16 @@ package org.elasticsearch.xpack.sql.util; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import static java.util.Collections.emptyList; public abstract class CollectionUtils { + + public static boolean isEmpty(Collection col) { + return col == null || col.isEmpty(); + } + @SuppressWarnings("unchecked") public static List combine(List left, List right) { if (right.isEmpty()) { diff --git a/sql/server/src/main/java/org/elasticsearch/xpack/sql/util/StringUtils.java b/sql/server/src/main/java/org/elasticsearch/xpack/sql/util/StringUtils.java index 9ba7258f7c0..f31677b761a 100644 --- a/sql/server/src/main/java/org/elasticsearch/xpack/sql/util/StringUtils.java +++ b/sql/server/src/main/java/org/elasticsearch/xpack/sql/util/StringUtils.java @@ -5,14 +5,19 @@ */ package org.elasticsearch.xpack.sql.util; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import java.io.IOException; import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.regex.Pattern; -import org.elasticsearch.common.Strings; - import static java.util.stream.Collectors.joining; public abstract class StringUtils { @@ -158,4 +163,13 @@ public abstract class StringUtils { public static Pattern likeRegex(String likePattern) { return Pattern.compile(sqlToJavaPattern(likePattern, '\\', true)); } + + public static String toString(SearchSourceBuilder source) { + try (XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint().humanReadable(true)) { + source.toXContent(builder, ToXContent.EMPTY_PARAMS); + return builder.string(); + } catch (IOException e) { + throw new RuntimeException("error rendering", e); + } + } } \ No newline at end of file diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/ProcessingHitExtractorTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/ProcessingHitExtractorTests.java deleted file mode 100644 index 3588fa7dd78..00000000000 --- a/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/ProcessingHitExtractorTests.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.sql.execution.search; - -import org.elasticsearch.common.document.DocumentField; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.io.stream.Writeable.Reader; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xpack.sql.expression.function.scalar.CastProcessorTests; -import org.elasticsearch.xpack.sql.expression.function.scalar.ColumnProcessor; -import org.elasticsearch.xpack.sql.expression.function.scalar.ComposeProcessorTests; -import org.elasticsearch.xpack.sql.expression.function.scalar.DateTimeProcessorTests; -import org.elasticsearch.xpack.sql.expression.function.scalar.MathFunctionProcessor; -import org.elasticsearch.xpack.sql.expression.function.scalar.MathFunctionProcessorTests; -import org.elasticsearch.xpack.sql.expression.function.scalar.MatrixFieldProcessorTests; -import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.function.Supplier; - -import static java.util.Collections.singletonList; -import static java.util.Collections.singletonMap; - -public class ProcessingHitExtractorTests extends AbstractWireSerializingTestCase { - public static ProcessingHitExtractor randomProcessingHitExtractor(int depth) { - return new ProcessingHitExtractor(ScrollCursorTests.randomHitExtractor(depth + 1), randomColumnProcessor(0)); - } - - public static ColumnProcessor randomColumnProcessor(int depth) { - List> options = new ArrayList<>(); - if (depth < 5) { - options.add(() -> ComposeProcessorTests.randomComposeProcessor(depth)); - } - options.add(CastProcessorTests::randomCastProcessor); - options.add(DateTimeProcessorTests::randomDateTimeProcessor); - options.add(MathFunctionProcessorTests::randomMathFunctionProcessor); - options.add(MatrixFieldProcessorTests::randomMatrixFieldProcessor); - return randomFrom(options).get(); - } - - @Override - protected NamedWriteableRegistry getNamedWriteableRegistry() { - return new NamedWriteableRegistry(HitExtractor.getNamedWriteables()); - } - - @Override - protected ProcessingHitExtractor createTestInstance() { - return randomProcessingHitExtractor(0); - } - - @Override - protected Reader instanceReader() { - return ProcessingHitExtractor::new; - } - - @Override - protected ProcessingHitExtractor mutateInstance(ProcessingHitExtractor instance) throws IOException { - @SuppressWarnings("unchecked") - Supplier supplier = randomFrom( - () -> new ProcessingHitExtractor( - randomValueOtherThan(instance.delegate(), () -> ScrollCursorTests.randomHitExtractor(0)), - instance.processor()), - () -> new ProcessingHitExtractor( - instance.delegate(), - randomValueOtherThan(instance.processor(), () -> randomColumnProcessor(0)))); - return supplier.get(); - } - - public void testGet() { - String fieldName = randomAlphaOfLength(5); - ProcessingHitExtractor extractor = new ProcessingHitExtractor( - new DocValueExtractor(fieldName), new MathFunctionProcessor(MathProcessor.LOG)); - - int times = between(1, 1000); - for (int i = 0; i < times; i++) { - double value = randomDouble(); - double expected = Math.log(value); - SearchHit hit = new SearchHit(1); - DocumentField field = new DocumentField(fieldName, singletonList(value)); - hit.fields(singletonMap(fieldName, field)); - assertEquals(expected, extractor.get(hit)); - } - } -} diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/ScrollCursorTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/ScrollCursorTests.java index 57b12eb7e22..44022ee51da 100644 --- a/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/ScrollCursorTests.java +++ b/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/ScrollCursorTests.java @@ -10,6 +10,13 @@ import org.elasticsearch.common.io.FastStringReader; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.sql.execution.search.extractor.ConstantExtractorTests; +import org.elasticsearch.xpack.sql.execution.search.extractor.DocValueExtractorTests; +import org.elasticsearch.xpack.sql.execution.search.extractor.HitExtractor; +import org.elasticsearch.xpack.sql.execution.search.extractor.HitExtractors; +import org.elasticsearch.xpack.sql.execution.search.extractor.InnerHitExtractorTests; +import org.elasticsearch.xpack.sql.execution.search.extractor.ProcessingHitExtractorTests; +import org.elasticsearch.xpack.sql.execution.search.extractor.SourceExtractorTests; import java.io.IOException; import java.io.StringWriter; @@ -41,7 +48,7 @@ public class ScrollCursorTests extends AbstractWireSerializingTestCase { - static ConstantExtractor randomConstantExtractor() { + public static ConstantExtractor randomConstantExtractor() { return new ConstantExtractor(randomValidConstant()); } diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/DocValueExtractorTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/DocValueExtractorTests.java similarity index 94% rename from sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/DocValueExtractorTests.java rename to sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/DocValueExtractorTests.java index 51de2f0ee17..0a1a3e430e9 100644 --- a/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/DocValueExtractorTests.java +++ b/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/DocValueExtractorTests.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.sql.execution.search; +package org.elasticsearch.xpack.sql.execution.search.extractor; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.io.stream.Writeable.Reader; @@ -17,7 +17,7 @@ import java.util.List; import static java.util.Collections.singletonMap; public class DocValueExtractorTests extends AbstractWireSerializingTestCase { - static DocValueExtractor randomDocValueExtractor() { + public static DocValueExtractor randomDocValueExtractor() { return new DocValueExtractor(randomAlphaOfLength(5)); } diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/InnerHitExtractorTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/InnerHitExtractorTests.java similarity index 91% rename from sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/InnerHitExtractorTests.java rename to sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/InnerHitExtractorTests.java index f6661bc7815..099c682bb4d 100644 --- a/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/InnerHitExtractorTests.java +++ b/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/InnerHitExtractorTests.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.sql.execution.search; +package org.elasticsearch.xpack.sql.execution.search.extractor; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.test.AbstractWireSerializingTestCase; @@ -11,7 +11,7 @@ import org.elasticsearch.test.AbstractWireSerializingTestCase; import java.io.IOException; public class InnerHitExtractorTests extends AbstractWireSerializingTestCase { - static InnerHitExtractor randomInnerHitExtractor() { + public static InnerHitExtractor randomInnerHitExtractor() { return new InnerHitExtractor(randomAlphaOfLength(5), randomAlphaOfLength(5), randomBoolean()); } diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ProcessingHitExtractorTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ProcessingHitExtractorTests.java new file mode 100644 index 00000000000..cda06186c62 --- /dev/null +++ b/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ProcessingHitExtractorTests.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.execution.search.extractor; + +import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.sql.expression.function.scalar.CastProcessorTests; +import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeProcessorTests; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathFunctionProcessorTests; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.ChainingProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.ChainingProcessorTests; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.HitExtractorProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.MatrixFieldProcessorTests; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; + +public class ProcessingHitExtractorTests extends AbstractWireSerializingTestCase { + public static ComputingHitExtractor randomProcessingHitExtractor(int depth) { + return new ComputingHitExtractor(randomProcessor(0)); + } + + public static Processor randomProcessor(int depth) { + List> options = new ArrayList<>(); + if (depth < 5) { + options.add(() -> ChainingProcessorTests.randomComposeProcessor(depth)); + } + options.add(CastProcessorTests::randomCastProcessor); + options.add(DateTimeProcessorTests::randomDateTimeProcessor); + options.add(MathFunctionProcessorTests::randomMathFunctionProcessor); + options.add(MatrixFieldProcessorTests::randomMatrixFieldProcessor); + return randomFrom(options).get(); + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(HitExtractors.getNamedWriteables()); + } + + @Override + protected ComputingHitExtractor createTestInstance() { + return randomProcessingHitExtractor(0); + } + + @Override + protected Reader instanceReader() { + return ComputingHitExtractor::new; + } + + @Override + protected ComputingHitExtractor mutateInstance(ComputingHitExtractor instance) throws IOException { + return new ComputingHitExtractor( + randomValueOtherThan(instance.processor(), () -> randomProcessor(0))); + } + + public void testGet() { + String fieldName = randomAlphaOfLength(5); + ChainingProcessor extractor = new ChainingProcessor(new HitExtractorProcessor(new DocValueExtractor(fieldName)), new MathProcessor(MathOperation.LOG)); + + int times = between(1, 1000); + for (int i = 0; i < times; i++) { + double value = randomDouble(); + double expected = Math.log(value); + SearchHit hit = new SearchHit(1); + DocumentField field = new DocumentField(fieldName, singletonList(value)); + hit.fields(singletonMap(fieldName, field)); + assertEquals(expected, extractor.process(hit)); + } + } +} diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceExtractorTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/SourceExtractorTests.java similarity index 95% rename from sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceExtractorTests.java rename to sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/SourceExtractorTests.java index ef30f65be7f..2341c86e2b9 100644 --- a/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceExtractorTests.java +++ b/sql/server/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/SourceExtractorTests.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.sql.execution.search; +package org.elasticsearch.xpack.sql.execution.search.extractor; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.Writeable.Reader; @@ -16,7 +16,7 @@ import java.io.IOException; import java.util.function.Supplier; public class SourceExtractorTests extends AbstractWireSerializingTestCase { - static SourceExtractor randomSourceExtractor() { + public static SourceExtractor randomSourceExtractor() { return new SourceExtractor(randomAlphaOfLength(5)); } diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/CastProcessorTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/CastProcessorTests.java index 3a74a9b8cda..831978705d0 100644 --- a/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/CastProcessorTests.java +++ b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/CastProcessorTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.sql.expression.function.scalar; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; @@ -35,16 +36,21 @@ public class CastProcessorTests extends AbstractWireSerializingTestCase proc.apply("1.2")); + assertEquals(null, proc.process(null)); + assertEquals(1, proc.process("1")); + Exception e = expectThrows(SqlIllegalArgumentException.class, () -> proc.process("1.2")); assertEquals("cannot cast [1.2] to [Int]", e.getMessage()); } { CastProcessor proc = new CastProcessor(Conversion.BOOL_TO_INT); - assertEquals(null, proc.apply(null)); - assertEquals(1, proc.apply(true)); - assertEquals(0, proc.apply(false)); + assertEquals(null, proc.process(null)); + assertEquals(1, proc.process(true)); + assertEquals(0, proc.process(false)); } } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(Processors.getNamedWriteables()); + } } diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/ComposeProcessorTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/ComposeProcessorTests.java deleted file mode 100644 index 9a3c400a6e2..00000000000 --- a/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/ComposeProcessorTests.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.sql.expression.function.scalar; - -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.io.stream.Writeable.Reader; -import org.elasticsearch.test.AbstractWireSerializingTestCase; - -import java.io.IOException; -import java.util.function.Supplier; - -import static org.elasticsearch.xpack.sql.execution.search.ProcessingHitExtractorTests.randomColumnProcessor; - -public class ComposeProcessorTests extends AbstractWireSerializingTestCase { - public static ComposeProcessor randomComposeProcessor(int depth) { - return new ComposeProcessor(randomColumnProcessor(depth + 1), randomColumnProcessor(depth + 1)); - } - - @Override - protected NamedWriteableRegistry getNamedWriteableRegistry() { - return new NamedWriteableRegistry(ColumnProcessor.getNamedWriteables()); - } - - @Override - protected ComposeProcessor createTestInstance() { - return randomComposeProcessor(0); - } - - @Override - protected Reader instanceReader() { - return ComposeProcessor::new; - } - - @Override - protected ComposeProcessor mutateInstance(ComposeProcessor instance) throws IOException { - @SuppressWarnings("unchecked") - Supplier supplier = randomFrom( - () -> new ComposeProcessor( - instance.first(), randomValueOtherThan(instance.second(), () -> randomColumnProcessor(0))), - () -> new ComposeProcessor( - randomValueOtherThan(instance.first(), () -> randomColumnProcessor(0)), instance.second())); - return supplier.get(); - } -} diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/MathFunctionProcessorTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/MathFunctionProcessorTests.java deleted file mode 100644 index 28f0fa3b483..00000000000 --- a/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/MathFunctionProcessorTests.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.sql.expression.function.scalar; - -import org.elasticsearch.common.io.stream.Writeable.Reader; -import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor; - -import java.io.IOException; - -public class MathFunctionProcessorTests extends AbstractWireSerializingTestCase { - public static MathFunctionProcessor randomMathFunctionProcessor() { - return new MathFunctionProcessor(randomFrom(MathProcessor.values())); - } - - @Override - protected MathFunctionProcessor createTestInstance() { - return randomMathFunctionProcessor(); - } - - @Override - protected Reader instanceReader() { - return MathFunctionProcessor::new; - } - - @Override - protected MathFunctionProcessor mutateInstance(MathFunctionProcessor instance) throws IOException { - return new MathFunctionProcessor(randomValueOtherThan(instance.processor(), () -> randomFrom(MathProcessor.values()))); - } - - public void testApply() { - MathFunctionProcessor proc = new MathFunctionProcessor(MathProcessor.E); - assertEquals(Math.E, proc.apply(null)); - assertEquals(Math.E, proc.apply("cat")); - assertEquals(Math.E, proc.apply(Math.PI)); - - proc = new MathFunctionProcessor(MathProcessor.SQRT); - assertEquals(2.0, (double) proc.apply(4), 0); - assertEquals(3.0, (double) proc.apply(9d), 0); - assertEquals(1.77, (double) proc.apply(3.14), 0.01); - } -} diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/BinaryArithmeticProcessorTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/BinaryArithmeticProcessorTests.java new file mode 100644 index 00000000000..de69e44804c --- /dev/null +++ b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/arithmetic/BinaryArithmeticProcessorTests.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.Literal; +import org.elasticsearch.xpack.sql.expression.function.scalar.Processors; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.ConstantProcessor; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; + +import static org.elasticsearch.xpack.sql.tree.Location.EMPTY; + +public class BinaryArithmeticProcessorTests extends AbstractWireSerializingTestCase { + public static BinaryArithmeticProcessor randomProcessor() { + return new BinaryArithmeticProcessor( + new ConstantProcessor(randomLong()), + new ConstantProcessor(randomLong()), + randomFrom(BinaryArithmeticProcessor.BinaryArithmeticOperation.values())); + } + + @Override + protected BinaryArithmeticProcessor createTestInstance() { + return randomProcessor(); + } + + @Override + protected Reader instanceReader() { + return BinaryArithmeticProcessor::new; + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(Processors.getNamedWriteables()); + } + + public void testAdd() { + BinaryArithmeticProcessor ba = new Add(EMPTY, l(7), l(3)).makeProcessor().asProcessor(); + assertEquals(10, ba.process(null)); + } + + public void testSub() { + BinaryArithmeticProcessor ba = new Sub(EMPTY, l(7), l(3)).makeProcessor().asProcessor(); + assertEquals(4, ba.process(null)); + } + + public void testMul() { + BinaryArithmeticProcessor ba = new Mul(EMPTY, l(7), l(3)).makeProcessor().asProcessor(); + assertEquals(21, ba.process(null)); + } + + public void testDiv() { + BinaryArithmeticProcessor ba = new Div(EMPTY, l(7), l(3)).makeProcessor().asProcessor(); + assertEquals(2, ((Number) ba.process(null)).longValue()); + ba = new Div(EMPTY, l((double) 7), l(3)).makeProcessor().asProcessor(); + assertEquals(2.33, ((Number) ba.process(null)).doubleValue(), 0.01d); + } + + public void testMod() { + BinaryArithmeticProcessor ba = new Mod(EMPTY, l(7), l(3)).makeProcessor().asProcessor(); + assertEquals(1, ba.process(null)); + } + + public void testNegate() { + Processor ba = new Neg(EMPTY, l(7)).asProcessor().asProcessor(); + assertEquals(-7, ba.process(null)); + } + + // ((3*2+4)/2-2)%2 + public void testTree() { + Expression mul = new Mul(EMPTY, l(3), l(2)); + Expression add = new Add(EMPTY, mul, l(4)); + Expression div = new Div(EMPTY, add, l(2)); + Expression sub = new Sub(EMPTY, div, l(2)); + Mod mod = new Mod(EMPTY, sub, l(2)); + + Processor proc = mod.makeProcessor().asProcessor(); + assertEquals(1, proc.process(null)); + } + + private static Literal l(Object value) { + return Literal.of(EMPTY, value); + } +} \ No newline at end of file diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DateTimeProcessorTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeProcessorTests.java similarity index 67% rename from sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DateTimeProcessorTests.java rename to sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeProcessorTests.java index 60f2cbe0d69..70ccad04087 100644 --- a/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DateTimeProcessorTests.java +++ b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeProcessorTests.java @@ -3,11 +3,11 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.sql.expression.function.scalar; +package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeExtractor; +import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeProcessor.DateTimeExtractor; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -30,20 +30,19 @@ public class DateTimeProcessorTests extends AbstractWireSerializingTestCase randomFrom(DateTimeExtractor.values())), - DateTimeZone.UTC); + return new DateTimeProcessor(randomValueOtherThan(instance.extractor(), () -> randomFrom(DateTimeExtractor.values())), DateTimeZone.UTC); } public void testApply() { DateTimeProcessor proc = new DateTimeProcessor(DateTimeExtractor.YEAR, DateTimeZone.UTC); - assertEquals(1970, proc.apply(0L)); - assertEquals(1970, proc.apply(new DateTime(0L, DateTimeZone.UTC))); - assertEquals(2017, proc.apply(new DateTime(2017, 01, 02, 10, 10, DateTimeZone.UTC))); + assertEquals(1970, proc.process(0L)); + assertEquals(1970, proc.process(new DateTime(0L, DateTimeZone.UTC))); + assertEquals(2017, proc.process(new DateTime(2017, 01, 02, 10, 10, DateTimeZone.UTC))); proc = new DateTimeProcessor(DateTimeExtractor.DAY_OF_MONTH, DateTimeZone.UTC); - assertEquals(1, proc.apply(0L)); - assertEquals(1, proc.apply(new DateTime(0L, DateTimeZone.UTC))); - assertEquals(2, proc.apply(new DateTime(2017, 01, 02, 10, 10, DateTimeZone.UTC))); - assertEquals(31, proc.apply(new DateTime(2017, 01, 31, 10, 10, DateTimeZone.UTC))); + assertEquals(1, proc.process(0L)); + assertEquals(1, proc.process(new DateTime(0L, DateTimeZone.UTC))); + assertEquals(2, proc.process(new DateTime(2017, 01, 02, 10, 10, DateTimeZone.UTC))); + assertEquals(31, proc.process(new DateTime(2017, 01, 31, 10, 10, DateTimeZone.UTC))); } } diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DayOfYearTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DayOfYearTests.java index 21a5f682563..f6722281df9 100644 --- a/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DayOfYearTests.java +++ b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DayOfYearTests.java @@ -25,7 +25,7 @@ public class DayOfYearTests extends ESTestCase { } private Object extract(Object value, DateTimeZone timeZone) { - return build(value, timeZone).asProcessor().apply(value); + return build(value, timeZone).asProcessor().asProcessor().process(value); } private DayOfYear build(Object value, DateTimeZone timeZone) { diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathFunctionProcessorTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathFunctionProcessorTests.java new file mode 100644 index 00000000000..886531f6f94 --- /dev/null +++ b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathFunctionProcessorTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.math; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; + +import java.io.IOException; + +public class MathFunctionProcessorTests extends AbstractWireSerializingTestCase { + public static MathProcessor randomMathFunctionProcessor() { + return new MathProcessor(randomFrom(MathOperation.values())); + } + + @Override + protected MathProcessor createTestInstance() { + return randomMathFunctionProcessor(); + } + + @Override + protected Reader instanceReader() { + return MathProcessor::new; + } + + @Override + protected MathProcessor mutateInstance(MathProcessor instance) throws IOException { + return new MathProcessor(randomValueOtherThan(instance.processor(), () -> randomFrom(MathOperation.values()))); + } + + public void testApply() { + MathProcessor proc = new MathProcessor(MathOperation.E); + assertEquals(Math.E, proc.process(null)); + assertEquals(Math.E, proc.process("cat")); + assertEquals(Math.E, proc.process(Math.PI)); + + proc = new MathProcessor(MathOperation.SQRT); + assertEquals(2.0, (double) proc.process(4), 0); + assertEquals(3.0, (double) proc.process(9d), 0); + assertEquals(1.77, (double) proc.process(3.14), 0.01); + } +} diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/ChainingProcessorTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/ChainingProcessorTests.java new file mode 100644 index 00000000000..ca261478a7c --- /dev/null +++ b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/ChainingProcessorTests.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.sql.expression.function.scalar.Processors; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.ChainingProcessor; + +import java.io.IOException; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.sql.execution.search.extractor.ProcessingHitExtractorTests.randomProcessor; + +public class ChainingProcessorTests extends AbstractWireSerializingTestCase { + public static ChainingProcessor randomComposeProcessor(int depth) { + return new ChainingProcessor(randomProcessor(depth + 1), randomProcessor(depth + 1)); + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(Processors.getNamedWriteables()); + } + + @Override + protected ChainingProcessor createTestInstance() { + return randomComposeProcessor(0); + } + + @Override + protected Reader instanceReader() { + return ChainingProcessor::new; + } + + @Override + protected ChainingProcessor mutateInstance(ChainingProcessor instance) throws IOException { + @SuppressWarnings("unchecked") + Supplier supplier = randomFrom( + () -> new ChainingProcessor( + instance.first(), randomValueOtherThan(instance.second(), () -> randomProcessor(0))), + () -> new ChainingProcessor( + randomValueOtherThan(instance.first(), () -> randomProcessor(0)), instance.second())); + return supplier.get(); + } +} diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/ConstantProcessorTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/ConstantProcessorTests.java new file mode 100644 index 00000000000..b26cf8edc1c --- /dev/null +++ b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/ConstantProcessorTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; + +public class ConstantProcessorTests extends AbstractWireSerializingTestCase { + public static ConstantProcessor randomConstantProcessor() { + return new ConstantProcessor(randomAlphaOfLength(5)); + } + + @Override + protected ConstantProcessor createTestInstance() { + return randomConstantProcessor(); + } + + @Override + protected Reader instanceReader() { + return ConstantProcessor::new; + } + + @Override + protected ConstantProcessor mutateInstance(ConstantProcessor instance) throws IOException { + return new ConstantProcessor(randomValueOtherThan(instance.process(null), () -> randomAlphaOfLength(5))); + } + + public void testApply() { + ConstantProcessor proc = new ConstantProcessor("test"); + assertEquals("test", proc.process(null)); + assertEquals("test", proc.process("cat")); + } +} diff --git a/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/MatrixFieldProcessorTests.java b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/MatrixFieldProcessorTests.java similarity index 80% rename from sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/MatrixFieldProcessorTests.java rename to sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/MatrixFieldProcessorTests.java index e0e0477d868..6763c41eb3c 100644 --- a/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/MatrixFieldProcessorTests.java +++ b/sql/server/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/processor/runtime/MatrixFieldProcessorTests.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.sql.expression.function.scalar; +package org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.test.AbstractWireSerializingTestCase; @@ -34,9 +34,9 @@ public class MatrixFieldProcessorTests extends AbstractWireSerializingTestCase conversion.convert("0xff")); - assertEquals("cannot cast [0xff] to [Date]: Invalid format: \"0xff\" is malformed at \"xff\"", e.getMessage()); + assertEquals("cannot cast [0xff] to [Date]:Invalid format: \"0xff\" is malformed at \"xff\"", e.getMessage()); } }