diff --git a/core/src/main/java/org/apache/druid/math/expr/ExprEval.java b/core/src/main/java/org/apache/druid/math/expr/ExprEval.java index 8c234877083..c72be2e27e3 100644 --- a/core/src/main/java/org/apache/druid/math/expr/ExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/ExprEval.java @@ -1303,6 +1303,9 @@ public abstract class ExprEval return null; } return Arrays.stream(value).map(value -> { + if (value == null) { + return null; + } Long lv = GuavaUtils.tryParseLong(value); if (lv == null) { Double d = Doubles.tryParse(value); @@ -1320,7 +1323,12 @@ public abstract class ExprEval if (value == null) { return null; } - return Arrays.stream(value).map(Doubles::tryParse).toArray(Double[]::new); + return Arrays.stream(value).map(val -> { + if (val == null) { + return null; + } + return Doubles.tryParse(val); + }).toArray(Double[]::new); } } } diff --git a/core/src/test/java/org/apache/druid/math/expr/ExprEvalTest.java b/core/src/test/java/org/apache/druid/math/expr/ExprEvalTest.java index ae8b5e3e2ee..3ce0d0cb319 100644 --- a/core/src/test/java/org/apache/druid/math/expr/ExprEvalTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/ExprEvalTest.java @@ -259,6 +259,20 @@ public class ExprEvalTest extends InitializedNullHandlingTest ); } + @Test + public void testStringArrayToNumberArray() + { + ExprEval someStringArray = ExprEval.ofStringArray(new String[]{"1", "2", "foo", null, "3.3"}); + Assert.assertArrayEquals( + new Long[]{1L, 2L, null, null, 3L}, + someStringArray.asLongArray() + ); + Assert.assertArrayEquals( + new Double[]{1.0, 2.0, null, null, 3.3}, + someStringArray.asDoubleArray() + ); + } + private void assertExpr(int position, Object expected) { assertExpr(position, ExprEval.bestEffortOf(expected)); diff --git a/docs/querying/sql.md b/docs/querying/sql.md index 1fbed13b96d..6f837de8ec1 100644 --- a/docs/querying/sql.md +++ b/docs/querying/sql.md @@ -1022,6 +1022,7 @@ Connection context can be specified as JDBC connection properties or as a "conte |---------|-----------|-------------| |`sqlQueryId`|Unique identifier given to this SQL query. For HTTP client, it will be returned in `X-Druid-SQL-Query-Id` header.|auto-generated| |`sqlTimeZone`|Sets the time zone for this connection, which will affect how time functions and timestamp literals behave. Should be a time zone name like "America/Los_Angeles" or offset like "-08:00".|druid.sql.planner.sqlTimeZone on the Broker (default: UTC)| +|`sqlStringifyArrays`|When set to true, result columns which return array values will be serialized into a JSON string in the response instead of as an array (default: true, except for JDBC connections, where it is always false)| |`useApproximateCountDistinct`|Whether to use an approximate cardinality algorithm for `COUNT(DISTINCT foo)`.|druid.sql.planner.useApproximateCountDistinct on the Broker (default: true)| |`useGroupingSetForExactDistinct`|Whether to use grouping sets to execute queries with multiple exact distinct aggregations.|druid.sql.planner.useGroupingSetForExactDistinct on the Broker (default: false)| |`useApproximateTopN`|Whether to use approximate [TopN queries](topnquery.md) when a SQL query could be expressed as such. If false, exact [GroupBy queries](groupbyquery.md) will be used instead.|druid.sql.planner.useApproximateTopN on the Broker (default: true)| diff --git a/sql/src/main/java/org/apache/druid/sql/avatica/DruidMeta.java b/sql/src/main/java/org/apache/druid/sql/avatica/DruidMeta.java index 145790f6f1b..c6862ffb0e5 100644 --- a/sql/src/main/java/org/apache/druid/sql/avatica/DruidMeta.java +++ b/sql/src/main/java/org/apache/druid/sql/avatica/DruidMeta.java @@ -45,6 +45,7 @@ import org.apache.druid.server.security.AuthenticatorMapper; import org.apache.druid.server.security.ForbiddenException; import org.apache.druid.sql.SqlLifecycleFactory; import org.apache.druid.sql.calcite.planner.Calcites; +import org.apache.druid.sql.calcite.planner.PlannerContext; import org.joda.time.Interval; import javax.annotation.Nonnull; @@ -109,6 +110,8 @@ public class DruidMeta extends MetaImpl context.put(entry); } } + // we don't want to stringify arrays for JDBC ever because avatica needs to handle this + context.put(PlannerContext.CTX_SQL_STRINGIFY_ARRAYS, false); openDruidConnection(ch.id, context.build()); } diff --git a/sql/src/main/java/org/apache/druid/sql/avatica/DruidStatement.java b/sql/src/main/java/org/apache/druid/sql/avatica/DruidStatement.java index 98379a3544e..ff4f2681016 100644 --- a/sql/src/main/java/org/apache/druid/sql/avatica/DruidStatement.java +++ b/sql/src/main/java/org/apache/druid/sql/avatica/DruidStatement.java @@ -28,6 +28,7 @@ import org.apache.calcite.avatica.Meta; import org.apache.calcite.avatica.remote.TypedValue; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.concurrent.Execs; @@ -114,12 +115,29 @@ public class DruidStatement implements Closeable for (int i = 0; i < fieldList.size(); i++) { RelDataTypeField field = fieldList.get(i); - final ColumnMetaData.Rep rep = QueryMaker.rep(field.getType().getSqlTypeName()); - final ColumnMetaData.ScalarType columnType = ColumnMetaData.scalar( - field.getType().getSqlTypeName().getJdbcOrdinal(), - field.getType().getSqlTypeName().getName(), - rep - ); + + final ColumnMetaData.AvaticaType columnType; + if (field.getType().getSqlTypeName() == SqlTypeName.ARRAY) { + final ColumnMetaData.Rep elementRep = QueryMaker.rep(field.getType().getComponentType().getSqlTypeName()); + final ColumnMetaData.ScalarType elementType = ColumnMetaData.scalar( + field.getType().getComponentType().getSqlTypeName().getJdbcOrdinal(), + field.getType().getComponentType().getSqlTypeName().getName(), + elementRep + ); + final ColumnMetaData.Rep arrayRep = QueryMaker.rep(field.getType().getSqlTypeName()); + columnType = ColumnMetaData.array( + elementType, + field.getType().getSqlTypeName().getName(), + arrayRep + ); + } else { + final ColumnMetaData.Rep rep = QueryMaker.rep(field.getType().getSqlTypeName()); + columnType = ColumnMetaData.scalar( + field.getType().getSqlTypeName().getJdbcOrdinal(), + field.getType().getSqlTypeName().getName(), + rep + ); + } columns.add( new ColumnMetaData( i, // ordinal diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java index 9aba1b7b86b..6f060f19c92 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java @@ -257,7 +257,7 @@ public class OperatorConversions /** * Sets the return type of the operator to "typeName", marked as non-nullable. * - * One of {@link #returnTypeNonNull}, {@link #returnTypeNullable}, or + * One of {@link #returnTypeNonNull}, {@link #returnTypeNullable}, {@link #returnTypeNullableArray}, or * {@link #returnTypeInference(SqlReturnTypeInference)} must be used before calling {@link #build()}. These methods * cannot be mixed; you must call exactly one. */ @@ -274,7 +274,7 @@ public class OperatorConversions /** * Sets the return type of the operator to "typeName", marked as nullable. * - * One of {@link #returnTypeNonNull}, {@link #returnTypeNullable}, or + * One of {@link #returnTypeNonNull}, {@link #returnTypeNullable}, {@link #returnTypeNullableArray}, or * {@link #returnTypeInference(SqlReturnTypeInference)} must be used before calling {@link #build()}. These methods * cannot be mixed; you must call exactly one. */ @@ -287,11 +287,28 @@ public class OperatorConversions ); return this; } + /** + * Sets the return type of the operator to an array type with elements of "typeName", marked as nullable. + * + * One of {@link #returnTypeNonNull}, {@link #returnTypeNullable}, {@link #returnTypeNullableArray}, or + * {@link #returnTypeInference(SqlReturnTypeInference)} must be used before calling {@link #build()}. These methods + * cannot be mixed; you must call exactly one. + */ + public OperatorBuilder returnTypeNullableArray(final SqlTypeName elementTypeName) + { + Preconditions.checkState(this.returnTypeInference == null, "Cannot set return type multiple times"); + + this.returnTypeInference = ReturnTypes.explicit( + factory -> Calcites.createSqlArrayTypeWithNullability(factory, elementTypeName, true) + ); + return this; + } + /** * Provides customized return type inference logic. * - * One of {@link #returnTypeNonNull}, {@link #returnTypeNullable}, or + * One of {@link #returnTypeNonNull}, {@link #returnTypeNullable}, {@link #returnTypeNullableArray}, or * {@link #returnTypeInference(SqlReturnTypeInference)} must be used before calling {@link #build()}. These methods * cannot be mixed; you must call exactly one. */ diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringAppendOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayAppendOperatorConversion.java similarity index 86% rename from sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringAppendOperatorConversion.java rename to sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayAppendOperatorConversion.java index d57d0fcb1a4..0c1aee2060d 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringAppendOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayAppendOperatorConversion.java @@ -24,22 +24,25 @@ import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlTypeFamily; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.sql.calcite.expression.DruidExpression; import org.apache.druid.sql.calcite.expression.OperatorConversions; import org.apache.druid.sql.calcite.expression.SqlOperatorConversion; import org.apache.druid.sql.calcite.planner.PlannerContext; -public class MultiValueStringAppendOperatorConversion implements SqlOperatorConversion +public class ArrayAppendOperatorConversion implements SqlOperatorConversion { private static final SqlFunction SQL_FUNCTION = OperatorConversions - .operatorBuilder("MV_APPEND") + .operatorBuilder("ARRAY_APPEND") .operandTypeChecker( OperandTypes.sequence( "(array,expr)", - OperandTypes.family(SqlTypeFamily.STRING), + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.ARRAY), + OperandTypes.family(SqlTypeFamily.STRING) + ), OperandTypes.or( OperandTypes.family(SqlTypeFamily.STRING), OperandTypes.family(SqlTypeFamily.NUMERIC) @@ -47,7 +50,7 @@ public class MultiValueStringAppendOperatorConversion implements SqlOperatorConv ) ) .functionCategory(SqlFunctionCategory.STRING) - .returnTypeNonNull(SqlTypeName.VARCHAR) + .returnTypeInference(ReturnTypes.ARG0_NULLABLE) .build(); @Override diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringConcatOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayConcatOperatorConversion.java similarity index 80% rename from sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringConcatOperatorConversion.java rename to sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayConcatOperatorConversion.java index 455715fe8f5..0f47c3d02b6 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringConcatOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayConcatOperatorConversion.java @@ -24,27 +24,33 @@ import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlTypeFamily; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.sql.calcite.expression.DruidExpression; import org.apache.druid.sql.calcite.expression.OperatorConversions; import org.apache.druid.sql.calcite.expression.SqlOperatorConversion; import org.apache.druid.sql.calcite.planner.PlannerContext; -public class MultiValueStringConcatOperatorConversion implements SqlOperatorConversion +public class ArrayConcatOperatorConversion implements SqlOperatorConversion { private static final SqlFunction SQL_FUNCTION = OperatorConversions - .operatorBuilder("MV_CONCAT") + .operatorBuilder("ARRAY_CONCAT") .operandTypeChecker( OperandTypes.sequence( "(array,array)", - OperandTypes.family(SqlTypeFamily.STRING), - OperandTypes.family(SqlTypeFamily.STRING) + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.ARRAY), + OperandTypes.family(SqlTypeFamily.STRING) + ), + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.ARRAY), + OperandTypes.family(SqlTypeFamily.STRING) + ) ) ) .functionCategory(SqlFunctionCategory.STRING) - .returnTypeNonNull(SqlTypeName.VARCHAR) + .returnTypeInference(ReturnTypes.ARG0_NULLABLE) .build(); @Override diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayContainsOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayContainsOperatorConversion.java index a4a9c0de815..705da9a0943 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayContainsOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayContainsOperatorConversion.java @@ -57,7 +57,8 @@ public class ArrayContainsOperatorConversion extends BaseExpressionDimFilterOper ), OperandTypes.or( OperandTypes.family(SqlTypeFamily.ARRAY), - OperandTypes.family(SqlTypeFamily.STRING) + OperandTypes.family(SqlTypeFamily.STRING), + OperandTypes.family(SqlTypeFamily.NUMERIC) ) ) ) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayLengthOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayLengthOperatorConversion.java index 3bcd6ec0de0..073d93556d8 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayLengthOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayLengthOperatorConversion.java @@ -38,9 +38,8 @@ public class ArrayLengthOperatorConversion implements SqlOperatorConversion .operatorBuilder("ARRAY_LENGTH") .operandTypeChecker( OperandTypes.or( - OperandTypes.family(SqlTypeFamily.STRING), OperandTypes.family(SqlTypeFamily.ARRAY), - OperandTypes.family(SqlTypeFamily.MULTISET) + OperandTypes.family(SqlTypeFamily.STRING) ) ) .functionCategory(SqlFunctionCategory.STRING) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOffsetOfOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOffsetOfOperatorConversion.java index 8d8da7461bb..51cad2feda4 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOffsetOfOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOffsetOfOperatorConversion.java @@ -40,9 +40,8 @@ public class ArrayOffsetOfOperatorConversion implements SqlOperatorConversion OperandTypes.sequence( "(array,expr)", OperandTypes.or( - OperandTypes.family(SqlTypeFamily.STRING), OperandTypes.family(SqlTypeFamily.ARRAY), - OperandTypes.family(SqlTypeFamily.MULTISET) + OperandTypes.family(SqlTypeFamily.STRING) ), OperandTypes.family(SqlTypeFamily.ANY) ) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOffsetOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOffsetOperatorConversion.java index 2fdee5d79b2..43a39d82be4 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOffsetOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOffsetOperatorConversion.java @@ -25,7 +25,6 @@ import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlTypeFamily; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.sql.calcite.expression.DruidExpression; import org.apache.druid.sql.calcite.expression.OperatorConversions; @@ -39,16 +38,15 @@ public class ArrayOffsetOperatorConversion implements SqlOperatorConversion .operandTypeChecker( OperandTypes.sequence( "(array,expr)", - OperandTypes.or( - OperandTypes.family(SqlTypeFamily.STRING), - OperandTypes.family(SqlTypeFamily.ARRAY), - OperandTypes.family(SqlTypeFamily.MULTISET) - ), + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.ARRAY), + OperandTypes.family(SqlTypeFamily.STRING) + ), OperandTypes.family(SqlTypeFamily.NUMERIC) ) ) .functionCategory(SqlFunctionCategory.STRING) - .returnTypeNonNull(SqlTypeName.VARCHAR) + .returnTypeInference(ArrayOrdinalOperatorConversion.ARG0_ELEMENT_INFERENCE) .build(); @Override diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOrdinalOfOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOrdinalOfOperatorConversion.java index 14fa9d61f3f..12edb572743 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOrdinalOfOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOrdinalOfOperatorConversion.java @@ -40,9 +40,8 @@ public class ArrayOrdinalOfOperatorConversion implements SqlOperatorConversion OperandTypes.sequence( "(array,expr)", OperandTypes.or( - OperandTypes.family(SqlTypeFamily.STRING), OperandTypes.family(SqlTypeFamily.ARRAY), - OperandTypes.family(SqlTypeFamily.MULTISET) + OperandTypes.family(SqlTypeFamily.STRING) ), OperandTypes.family(SqlTypeFamily.ANY) ) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOrdinalOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOrdinalOperatorConversion.java index c3d1302dc41..e9ec01a3b3c 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOrdinalOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOrdinalOperatorConversion.java @@ -19,13 +19,16 @@ package org.apache.druid.sql.calcite.expression.builtin; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlOperatorBinding; import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeFamily; -import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.sql.calcite.expression.DruidExpression; import org.apache.druid.sql.calcite.expression.OperatorConversions; @@ -34,21 +37,22 @@ import org.apache.druid.sql.calcite.planner.PlannerContext; public class ArrayOrdinalOperatorConversion implements SqlOperatorConversion { + static final SqlReturnTypeInference ARG0_ELEMENT_INFERENCE = new ArrayElementReturnTypeInference(); + private static final SqlFunction SQL_FUNCTION = OperatorConversions .operatorBuilder("ARRAY_ORDINAL") .operandTypeChecker( OperandTypes.sequence( "(array,expr)", OperandTypes.or( - OperandTypes.family(SqlTypeFamily.STRING), OperandTypes.family(SqlTypeFamily.ARRAY), - OperandTypes.family(SqlTypeFamily.MULTISET) + OperandTypes.family(SqlTypeFamily.STRING) ), OperandTypes.family(SqlTypeFamily.NUMERIC) ) ) .functionCategory(SqlFunctionCategory.STRING) - .returnTypeNonNull(SqlTypeName.VARCHAR) + .returnTypeInference(ARG0_ELEMENT_INFERENCE) .build(); @Override @@ -74,4 +78,17 @@ public class ArrayOrdinalOperatorConversion implements SqlOperatorConversion ) ); } + + static class ArrayElementReturnTypeInference implements SqlReturnTypeInference + { + @Override + public RelDataType inferReturnType(SqlOperatorBinding sqlOperatorBinding) + { + RelDataType type = sqlOperatorBinding.getOperandType(0); + if (SqlTypeUtil.isArray(type)) { + type.getComponentType(); + } + return type; + } + } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringPrependOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayPrependOperatorConversion.java similarity index 86% rename from sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringPrependOperatorConversion.java rename to sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayPrependOperatorConversion.java index 6fa04ce176c..06f270d6d52 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringPrependOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayPrependOperatorConversion.java @@ -24,18 +24,18 @@ import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlTypeFamily; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.sql.calcite.expression.DruidExpression; import org.apache.druid.sql.calcite.expression.OperatorConversions; import org.apache.druid.sql.calcite.expression.SqlOperatorConversion; import org.apache.druid.sql.calcite.planner.PlannerContext; -public class MultiValueStringPrependOperatorConversion implements SqlOperatorConversion +public class ArrayPrependOperatorConversion implements SqlOperatorConversion { private static final SqlFunction SQL_FUNCTION = OperatorConversions - .operatorBuilder("MV_PREPEND") + .operatorBuilder("ARRAY_PREPEND") .operandTypeChecker( OperandTypes.sequence( "(expr,array)", @@ -43,11 +43,14 @@ public class MultiValueStringPrependOperatorConversion implements SqlOperatorCon OperandTypes.family(SqlTypeFamily.STRING), OperandTypes.family(SqlTypeFamily.NUMERIC) ), - OperandTypes.family(SqlTypeFamily.STRING) + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.ARRAY), + OperandTypes.family(SqlTypeFamily.STRING) + ) ) ) .functionCategory(SqlFunctionCategory.STRING) - .returnTypeNonNull(SqlTypeName.VARCHAR) + .returnTypeInference(ReturnTypes.ARG1_NULLABLE) .build(); @Override diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringSliceOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArraySliceOperatorConversion.java similarity index 81% rename from sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringSliceOperatorConversion.java rename to sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArraySliceOperatorConversion.java index f278b3bf7eb..49b67015177 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringSliceOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArraySliceOperatorConversion.java @@ -24,35 +24,41 @@ import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlTypeFamily; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.sql.calcite.expression.DruidExpression; import org.apache.druid.sql.calcite.expression.OperatorConversions; import org.apache.druid.sql.calcite.expression.SqlOperatorConversion; import org.apache.druid.sql.calcite.planner.PlannerContext; -public class MultiValueStringSliceOperatorConversion implements SqlOperatorConversion +public class ArraySliceOperatorConversion implements SqlOperatorConversion { private static final SqlFunction SQL_FUNCTION = OperatorConversions - .operatorBuilder("MV_SLICE") + .operatorBuilder("ARRAY_SLICE") .operandTypeChecker( OperandTypes.or( OperandTypes.sequence( "(expr,start)", - OperandTypes.family(SqlTypeFamily.STRING), + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.ARRAY), + OperandTypes.family(SqlTypeFamily.STRING) + ), OperandTypes.family(SqlTypeFamily.NUMERIC) ), OperandTypes.sequence( "(expr,start,end)", - OperandTypes.family(SqlTypeFamily.STRING), + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.ARRAY), + OperandTypes.family(SqlTypeFamily.STRING) + ), OperandTypes.family(SqlTypeFamily.NUMERIC), OperandTypes.family(SqlTypeFamily.NUMERIC) ) ) ) .functionCategory(SqlFunctionCategory.STRING) - .returnTypeNonNull(SqlTypeName.VARCHAR) + .returnTypeInference(ReturnTypes.ARG0_NULLABLE) .build(); @Override diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayToStringOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayToStringOperatorConversion.java index 802a95e5a36..5d316a59116 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayToStringOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayToStringOperatorConversion.java @@ -40,9 +40,8 @@ public class ArrayToStringOperatorConversion implements SqlOperatorConversion OperandTypes.sequence( "(array,expr)", OperandTypes.or( - OperandTypes.family(SqlTypeFamily.STRING), OperandTypes.family(SqlTypeFamily.ARRAY), - OperandTypes.family(SqlTypeFamily.MULTISET) + OperandTypes.family(SqlTypeFamily.STRING) ), OperandTypes.family(SqlTypeFamily.ANY) ) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringOperatorConversions.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringOperatorConversions.java new file mode 100644 index 00000000000..cb6dcfdf5ce --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringOperatorConversions.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.expression.builtin; + +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.druid.sql.calcite.expression.AliasedOperatorConversion; +import org.apache.druid.sql.calcite.expression.OperatorConversions; + +/** + * Array functions which return an array, but are used in a multi-valued string dimension context instead will output + * {@link SqlTypeName#VARCHAR} instead of {@link SqlTypeName#ARRAY}. On the backend, these functions are identical, + * so these classes only override the signature information. + */ +public class MultiValueStringOperatorConversions +{ + public static class Append extends ArrayAppendOperatorConversion + { + private static final SqlFunction SQL_FUNCTION = OperatorConversions + .operatorBuilder("MV_APPEND") + .operandTypeChecker( + OperandTypes.sequence( + "(array,expr)", + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.ARRAY), + OperandTypes.family(SqlTypeFamily.STRING) + ), + OperandTypes.family(SqlTypeFamily.STRING) + ) + ) + .functionCategory(SqlFunctionCategory.STRING) + .returnTypeNullable(SqlTypeName.VARCHAR) + .build(); + + @Override + public SqlOperator calciteOperator() + { + return SQL_FUNCTION; + } + } + + public static class Prepend extends ArrayPrependOperatorConversion + { + private static final SqlFunction SQL_FUNCTION = OperatorConversions + .operatorBuilder("MV_PREPEND") + .operandTypeChecker( + OperandTypes.sequence( + "(expr,array)", + OperandTypes.family(SqlTypeFamily.STRING), + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.ARRAY), + OperandTypes.family(SqlTypeFamily.STRING) + ) + ) + ) + .functionCategory(SqlFunctionCategory.STRING) + .returnTypeNullable(SqlTypeName.VARCHAR) + .build(); + + @Override + public SqlOperator calciteOperator() + { + return SQL_FUNCTION; + } + } + + public static class Concat extends ArrayConcatOperatorConversion + { + private static final SqlFunction SQL_FUNCTION = OperatorConversions + .operatorBuilder("MV_CONCAT") + .operandTypeChecker( + OperandTypes.sequence( + "(array,array)", + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.ARRAY), + OperandTypes.family(SqlTypeFamily.STRING) + ), + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.ARRAY), + OperandTypes.family(SqlTypeFamily.STRING) + ) + ) + ) + .functionCategory(SqlFunctionCategory.STRING) + .returnTypeNullable(SqlTypeName.VARCHAR) + .build(); + + @Override + public SqlOperator calciteOperator() + { + return SQL_FUNCTION; + } + } + + public static class Contains extends ArrayContainsOperatorConversion + { + private static final SqlFunction SQL_FUNCTION = OperatorConversions + .operatorBuilder("MV_CONTAINS") + .operandTypeChecker( + OperandTypes.sequence( + "(array,array)", + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.ARRAY), + OperandTypes.family(SqlTypeFamily.STRING) + ), + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.ARRAY), + OperandTypes.family(SqlTypeFamily.STRING), + OperandTypes.family(SqlTypeFamily.NUMERIC) + ) + ) + ) + .returnTypeInference(ReturnTypes.BOOLEAN) + .build(); + + @Override + public SqlOperator calciteOperator() + { + return SQL_FUNCTION; + } + } + + public static class Offset extends ArrayOffsetOperatorConversion + { + private static final SqlFunction SQL_FUNCTION = OperatorConversions + .operatorBuilder("MV_OFFSET") + .operandTypeChecker( + OperandTypes.sequence( + "(array,expr)", + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.ARRAY), + OperandTypes.family(SqlTypeFamily.STRING) + ), + OperandTypes.family(SqlTypeFamily.NUMERIC) + ) + ) + .functionCategory(SqlFunctionCategory.STRING) + .returnTypeNullable(SqlTypeName.VARCHAR) + .build(); + + @Override + public SqlOperator calciteOperator() + { + return SQL_FUNCTION; + } + } + + public static class Ordinal extends ArrayOrdinalOperatorConversion + { + private static final SqlFunction SQL_FUNCTION = OperatorConversions + .operatorBuilder("MV_ORDINAL") + .operandTypeChecker( + OperandTypes.sequence( + "(array,expr)", + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.ARRAY), + OperandTypes.family(SqlTypeFamily.STRING) + ), + OperandTypes.family(SqlTypeFamily.NUMERIC) + ) + ) + .functionCategory(SqlFunctionCategory.STRING) + .returnTypeNullable(SqlTypeName.VARCHAR) + .build(); + + @Override + public SqlOperator calciteOperator() + { + return SQL_FUNCTION; + } + } + + public static class Slice extends ArraySliceOperatorConversion + { + private static final SqlFunction SQL_FUNCTION = OperatorConversions + .operatorBuilder("MV_SLICE") + .operandTypeChecker( + OperandTypes.or( + OperandTypes.sequence( + "(expr,start)", + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.ARRAY), + OperandTypes.family(SqlTypeFamily.STRING) + ), + OperandTypes.family(SqlTypeFamily.NUMERIC) + ), + OperandTypes.sequence( + "(expr,start,end)", + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.ARRAY), + OperandTypes.family(SqlTypeFamily.STRING) + ), + OperandTypes.family(SqlTypeFamily.NUMERIC), + OperandTypes.family(SqlTypeFamily.NUMERIC) + ) + ) + ) + .functionCategory(SqlFunctionCategory.STRING) + .returnTypeNullable(SqlTypeName.VARCHAR) + .build(); + + @Override + public SqlOperator calciteOperator() + { + return SQL_FUNCTION; + } + } + + public static class StringToMultiString extends StringToArrayOperatorConversion + { + private static final SqlFunction SQL_FUNCTION = OperatorConversions + .operatorBuilder("STRING_TO_MV") + .operandTypeChecker( + OperandTypes.sequence( + "(string,expr)", + OperandTypes.family(SqlTypeFamily.STRING), + OperandTypes.family(SqlTypeFamily.STRING) + ) + ) + .functionCategory(SqlFunctionCategory.STRING) + .returnTypeNullable(SqlTypeName.VARCHAR) + .build(); + + @Override + public SqlOperator calciteOperator() + { + return SQL_FUNCTION; + } + } + + public static class MultiStringToString extends AliasedOperatorConversion + { + public MultiStringToString() + { + super(new ArrayToStringOperatorConversion(), "MV_TO_STRING"); + } + } + + public static class Length extends AliasedOperatorConversion + { + public Length() + { + super(new ArrayLengthOperatorConversion(), "MV_LENGTH"); + } + } + + public static class OffsetOf extends AliasedOperatorConversion + { + public OffsetOf() + { + super(new ArrayOffsetOfOperatorConversion(), "MV_OFFSET_OF"); + } + } + + public static class OrdinalOf extends AliasedOperatorConversion + { + public OrdinalOf() + { + super(new ArrayOrdinalOfOperatorConversion(), "MV_ORDINAL_OF"); + } + } + + public static class Overlap extends AliasedOperatorConversion + { + public Overlap() + { + super(new ArrayOverlapOperatorConversion(), "MV_OVERLAP"); + } + } + + private MultiValueStringOperatorConversions() + { + // no instantiation + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/StringToMultiValueStringOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/StringToArrayOperatorConversion.java similarity index 93% rename from sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/StringToMultiValueStringOperatorConversion.java rename to sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/StringToArrayOperatorConversion.java index 7a6442f2dc8..9c6e064c916 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/StringToMultiValueStringOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/StringToArrayOperatorConversion.java @@ -32,11 +32,11 @@ import org.apache.druid.sql.calcite.expression.OperatorConversions; import org.apache.druid.sql.calcite.expression.SqlOperatorConversion; import org.apache.druid.sql.calcite.planner.PlannerContext; -public class StringToMultiValueStringOperatorConversion implements SqlOperatorConversion +public class StringToArrayOperatorConversion implements SqlOperatorConversion { // note: since this function produces an array private static final SqlFunction SQL_FUNCTION = OperatorConversions - .operatorBuilder("STRING_TO_MV") + .operatorBuilder("STRING_TO_ARRAY") .operandTypeChecker( OperandTypes.sequence( "(string,expr)", @@ -45,7 +45,7 @@ public class StringToMultiValueStringOperatorConversion implements SqlOperatorCo ) ) .functionCategory(SqlFunctionCategory.STRING) - .returnTypeNonNull(SqlTypeName.VARCHAR) + .returnTypeNullableArray(SqlTypeName.VARCHAR) .build(); @Override diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java index f0135b7c0a5..8b99dda7642 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java @@ -51,6 +51,8 @@ import org.apache.druid.sql.calcite.expression.SqlOperatorConversion; import org.apache.druid.sql.calcite.expression.UnaryFunctionOperatorConversion; import org.apache.druid.sql.calcite.expression.UnaryPrefixOperatorConversion; import org.apache.druid.sql.calcite.expression.UnarySuffixOperatorConversion; +import org.apache.druid.sql.calcite.expression.builtin.ArrayAppendOperatorConversion; +import org.apache.druid.sql.calcite.expression.builtin.ArrayConcatOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.ArrayConstructorOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.ArrayContainsOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.ArrayLengthOperatorConversion; @@ -59,6 +61,8 @@ import org.apache.druid.sql.calcite.expression.builtin.ArrayOffsetOperatorConver import org.apache.druid.sql.calcite.expression.builtin.ArrayOrdinalOfOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.ArrayOrdinalOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.ArrayOverlapOperatorConversion; +import org.apache.druid.sql.calcite.expression.builtin.ArrayPrependOperatorConversion; +import org.apache.druid.sql.calcite.expression.builtin.ArraySliceOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.ArrayToStringOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.BTrimOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.CastOperatorConversion; @@ -78,10 +82,7 @@ import org.apache.druid.sql.calcite.expression.builtin.LeastOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.LeftOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.LikeOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.MillisToTimestampOperatorConversion; -import org.apache.druid.sql.calcite.expression.builtin.MultiValueStringAppendOperatorConversion; -import org.apache.druid.sql.calcite.expression.builtin.MultiValueStringConcatOperatorConversion; -import org.apache.druid.sql.calcite.expression.builtin.MultiValueStringPrependOperatorConversion; -import org.apache.druid.sql.calcite.expression.builtin.MultiValueStringSliceOperatorConversion; +import org.apache.druid.sql.calcite.expression.builtin.MultiValueStringOperatorConversions; import org.apache.druid.sql.calcite.expression.builtin.ParseLongOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.PositionOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.RPadOperatorConversion; @@ -94,7 +95,7 @@ import org.apache.druid.sql.calcite.expression.builtin.ReverseOperatorConversion import org.apache.druid.sql.calcite.expression.builtin.RightOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.RoundOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.StringFormatOperatorConversion; -import org.apache.druid.sql.calcite.expression.builtin.StringToMultiValueStringOperatorConversion; +import org.apache.druid.sql.calcite.expression.builtin.StringToArrayOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.StrposOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.SubstringOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.TextcatOperatorConversion; @@ -123,19 +124,19 @@ public class DruidOperatorTable implements SqlOperatorTable { private static final List STANDARD_AGGREGATORS = ImmutableList.builder() - .add(new ApproxCountDistinctSqlAggregator()) - .add(new AvgSqlAggregator()) - .add(new CountSqlAggregator()) - .add(EarliestLatestAnySqlAggregator.EARLIEST) - .add(EarliestLatestAnySqlAggregator.LATEST) - .add(EarliestLatestAnySqlAggregator.ANY_VALUE) - .add(new MinSqlAggregator()) - .add(new MaxSqlAggregator()) - .add(new SumSqlAggregator()) - .add(new SumZeroSqlAggregator()) - .add(new GroupingSqlAggregator()) - .add(new ArraySqlAggregator()) - .build(); + .add(new ApproxCountDistinctSqlAggregator()) + .add(new AvgSqlAggregator()) + .add(new CountSqlAggregator()) + .add(EarliestLatestAnySqlAggregator.EARLIEST) + .add(EarliestLatestAnySqlAggregator.LATEST) + .add(EarliestLatestAnySqlAggregator.ANY_VALUE) + .add(new MinSqlAggregator()) + .add(new MaxSqlAggregator()) + .add(new SumSqlAggregator()) + .add(new SumZeroSqlAggregator()) + .add(new GroupingSqlAggregator()) + .add(new ArraySqlAggregator()) + .build(); // STRLEN has so many aliases. @@ -146,182 +147,199 @@ public class DruidOperatorTable implements SqlOperatorTable private static final List TIME_OPERATOR_CONVERSIONS = ImmutableList.builder() - .add(new CeilOperatorConversion()) - .add(new DateTruncOperatorConversion()) - .add(new ExtractOperatorConversion()) - .add(new FloorOperatorConversion()) - .add(new MillisToTimestampOperatorConversion()) - .add(new TimeArithmeticOperatorConversion.TimeMinusIntervalOperatorConversion()) - .add(new TimeArithmeticOperatorConversion.TimePlusIntervalOperatorConversion()) - .add(new TimeExtractOperatorConversion()) - .add(new TimeCeilOperatorConversion()) - .add(new TimeFloorOperatorConversion()) - .add(new TimeFormatOperatorConversion()) - .add(new TimeParseOperatorConversion()) - .add(new TimeShiftOperatorConversion()) - .add(new TimestampToMillisOperatorConversion()) - .build(); + .add(new CeilOperatorConversion()) + .add(new DateTruncOperatorConversion()) + .add(new ExtractOperatorConversion()) + .add(new FloorOperatorConversion()) + .add(new MillisToTimestampOperatorConversion()) + .add(new TimeArithmeticOperatorConversion.TimeMinusIntervalOperatorConversion()) + .add(new TimeArithmeticOperatorConversion.TimePlusIntervalOperatorConversion()) + .add(new TimeExtractOperatorConversion()) + .add(new TimeCeilOperatorConversion()) + .add(new TimeFloorOperatorConversion()) + .add(new TimeFormatOperatorConversion()) + .add(new TimeParseOperatorConversion()) + .add(new TimeShiftOperatorConversion()) + .add(new TimestampToMillisOperatorConversion()) + .build(); private static final List STRING_OPERATOR_CONVERSIONS = ImmutableList.builder() - .add(new BTrimOperatorConversion()) - .add(new LikeOperatorConversion()) - .add(new LTrimOperatorConversion()) - .add(new PositionOperatorConversion()) - .add(new RegexpExtractOperatorConversion()) - .add(new RegexpLikeOperatorConversion()) - .add(new RTrimOperatorConversion()) - .add(new ParseLongOperatorConversion()) - .add(new StringFormatOperatorConversion()) - .add(new StrposOperatorConversion()) - .add(new SubstringOperatorConversion()) - .add(new RightOperatorConversion()) - .add(new LeftOperatorConversion()) - .add(new ReverseOperatorConversion()) - .add(new RepeatOperatorConversion()) - .add(new AliasedOperatorConversion(new SubstringOperatorConversion(), "SUBSTR")) - .add(new ConcatOperatorConversion()) - .add(new TextcatOperatorConversion()) - .add(new TrimOperatorConversion()) - .add(new TruncateOperatorConversion()) - .add(new AliasedOperatorConversion(new TruncateOperatorConversion(), "TRUNC")) - .add(new LPadOperatorConversion()) - .add(new RPadOperatorConversion()) - .add(ContainsOperatorConversion.caseSensitive()) - .add(ContainsOperatorConversion.caseInsensitive()) - .build(); + .add(new BTrimOperatorConversion()) + .add(new LikeOperatorConversion()) + .add(new LTrimOperatorConversion()) + .add(new PositionOperatorConversion()) + .add(new RegexpExtractOperatorConversion()) + .add(new RegexpLikeOperatorConversion()) + .add(new RTrimOperatorConversion()) + .add(new ParseLongOperatorConversion()) + .add(new StringFormatOperatorConversion()) + .add(new StrposOperatorConversion()) + .add(new SubstringOperatorConversion()) + .add(new RightOperatorConversion()) + .add(new LeftOperatorConversion()) + .add(new ReverseOperatorConversion()) + .add(new RepeatOperatorConversion()) + .add(new AliasedOperatorConversion(new SubstringOperatorConversion(), "SUBSTR")) + .add(new ConcatOperatorConversion()) + .add(new TextcatOperatorConversion()) + .add(new TrimOperatorConversion()) + .add(new TruncateOperatorConversion()) + .add(new AliasedOperatorConversion(new TruncateOperatorConversion(), "TRUNC")) + .add(new LPadOperatorConversion()) + .add(new RPadOperatorConversion()) + .add(ContainsOperatorConversion.caseSensitive()) + .add(ContainsOperatorConversion.caseInsensitive()) + .build(); private static final List VALUE_COERCION_OPERATOR_CONVERSIONS = ImmutableList.builder() - .add(new CastOperatorConversion()) - .add(new ReinterpretOperatorConversion()) - .build(); + .add(new CastOperatorConversion()) + .add(new ReinterpretOperatorConversion()) + .build(); private static final List ARRAY_OPERATOR_CONVERSIONS = ImmutableList.builder() - .add(new ArrayConstructorOperatorConversion()) - .add(new ArrayContainsOperatorConversion()) - .add(new ArrayOverlapOperatorConversion()) - .add(new AliasedOperatorConversion(new ArrayContainsOperatorConversion(), "MV_CONTAINS")) - .add(new AliasedOperatorConversion(new ArrayOverlapOperatorConversion(), "MV_OVERLAP")) - .add(new ArrayLengthOperatorConversion()) - .add(new AliasedOperatorConversion(new ArrayLengthOperatorConversion(), "MV_LENGTH")) - .add(new ArrayOffsetOperatorConversion()) - .add(new AliasedOperatorConversion(new ArrayOffsetOperatorConversion(), "MV_OFFSET")) - .add(new ArrayOrdinalOperatorConversion()) - .add(new AliasedOperatorConversion(new ArrayOrdinalOperatorConversion(), "MV_ORDINAL")) - .add(new ArrayOffsetOfOperatorConversion()) - .add(new AliasedOperatorConversion(new ArrayOffsetOfOperatorConversion(), "MV_OFFSET_OF")) - .add(new ArrayOrdinalOfOperatorConversion()) - .add(new AliasedOperatorConversion(new ArrayOrdinalOfOperatorConversion(), "MV_ORDINAL_OF")) - .add(new ArrayToStringOperatorConversion()) - .add(new AliasedOperatorConversion(new ArrayToStringOperatorConversion(), "MV_TO_STRING")) - .build(); + .add(new ArrayConstructorOperatorConversion()) + .add(new ArrayContainsOperatorConversion()) + .add(new ArrayConcatOperatorConversion()) + .add(new ArrayOverlapOperatorConversion()) + .add(new ArrayAppendOperatorConversion()) + .add(new ArrayPrependOperatorConversion()) + .add(new ArrayLengthOperatorConversion()) + .add(new ArrayOffsetOperatorConversion()) + .add(new ArrayOrdinalOperatorConversion()) + .add(new ArrayOffsetOfOperatorConversion()) + .add(new ArrayOrdinalOfOperatorConversion()) + .add(new ArraySliceOperatorConversion()) + .add(new ArrayToStringOperatorConversion()) + .add(new StringToArrayOperatorConversion()) + .build(); private static final List MULTIVALUE_STRING_OPERATOR_CONVERSIONS = ImmutableList.builder() - .add(new MultiValueStringAppendOperatorConversion()) - .add(new MultiValueStringConcatOperatorConversion()) - .add(new MultiValueStringPrependOperatorConversion()) - .add(new MultiValueStringSliceOperatorConversion()) - .add(new StringToMultiValueStringOperatorConversion()) - .build(); + .add(new MultiValueStringOperatorConversions.Append()) + .add(new MultiValueStringOperatorConversions.Prepend()) + .add(new MultiValueStringOperatorConversions.Concat()) + .add(new MultiValueStringOperatorConversions.Contains()) + .add(new MultiValueStringOperatorConversions.Overlap()) + .add(new MultiValueStringOperatorConversions.Length()) + .add(new MultiValueStringOperatorConversions.Offset()) + .add(new MultiValueStringOperatorConversions.Ordinal()) + .add(new MultiValueStringOperatorConversions.OffsetOf()) + .add(new MultiValueStringOperatorConversions.OrdinalOf()) + .add(new MultiValueStringOperatorConversions.Slice()) + .add(new MultiValueStringOperatorConversions.MultiStringToString()) + .add(new MultiValueStringOperatorConversions.StringToMultiString()) + .build(); private static final List REDUCTION_OPERATOR_CONVERSIONS = ImmutableList.builder() - .add(new GreatestOperatorConversion()) - .add(new LeastOperatorConversion()) - .build(); + .add(new GreatestOperatorConversion()) + .add(new LeastOperatorConversion()) + .build(); private static final List IPV4ADDRESS_OPERATOR_CONVERSIONS = ImmutableList.builder() - .add(new IPv4AddressMatchOperatorConversion()) - .add(new IPv4AddressParseOperatorConversion()) - .add(new IPv4AddressStringifyOperatorConversion()) - .build(); + .add(new IPv4AddressMatchOperatorConversion()) + .add(new IPv4AddressParseOperatorConversion()) + .add(new IPv4AddressStringifyOperatorConversion()) + .build(); private static final List BITWISE_OPERATOR_CONVERSIONS = ImmutableList.builder() - .add(OperatorConversions.druidBinaryLongFn("BITWISE_AND", "bitwiseAnd")) - .add(OperatorConversions.druidUnaryLongFn("BITWISE_COMPLEMENT", "bitwiseComplement")) - .add(OperatorConversions.druidBinaryLongFn("BITWISE_OR", "bitwiseOr")) - .add(OperatorConversions.druidBinaryLongFn("BITWISE_SHIFT_LEFT", "bitwiseShiftLeft")) - .add(OperatorConversions.druidBinaryLongFn("BITWISE_SHIFT_RIGHT", "bitwiseShiftRight")) - .add(OperatorConversions.druidBinaryLongFn("BITWISE_XOR", "bitwiseXor")) - .add( - OperatorConversions.druidUnaryLongFn( - "BITWISE_CONVERT_DOUBLE_TO_LONG_BITS", - "bitwiseConvertDoubleToLongBits" - ) - ) - .add( - OperatorConversions.druidUnaryDoubleFn( - "BITWISE_CONVERT_LONG_BITS_TO_DOUBLE", - "bitwiseConvertLongBitsToDouble" - ) - ) - .build(); + .add(OperatorConversions.druidBinaryLongFn("BITWISE_AND", "bitwiseAnd")) + .add(OperatorConversions.druidUnaryLongFn("BITWISE_COMPLEMENT", "bitwiseComplement")) + .add(OperatorConversions.druidBinaryLongFn("BITWISE_OR", "bitwiseOr")) + .add(OperatorConversions.druidBinaryLongFn("BITWISE_SHIFT_LEFT", "bitwiseShiftLeft")) + .add(OperatorConversions.druidBinaryLongFn("BITWISE_SHIFT_RIGHT", "bitwiseShiftRight")) + .add(OperatorConversions.druidBinaryLongFn("BITWISE_XOR", "bitwiseXor")) + .add( + OperatorConversions.druidUnaryLongFn( + "BITWISE_CONVERT_DOUBLE_TO_LONG_BITS", + "bitwiseConvertDoubleToLongBits" + ) + ) + .add( + OperatorConversions.druidUnaryDoubleFn( + "BITWISE_CONVERT_LONG_BITS_TO_DOUBLE", + "bitwiseConvertLongBitsToDouble" + ) + ) + .build(); private static final List STANDARD_OPERATOR_CONVERSIONS = ImmutableList.builder() - .add(new DirectOperatorConversion(SqlStdOperatorTable.ABS, "abs")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.CASE, "case_searched")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.CHAR_LENGTH, "strlen")) - .add(CHARACTER_LENGTH_CONVERSION) - .add(new AliasedOperatorConversion(CHARACTER_LENGTH_CONVERSION, "LENGTH")) - .add(new AliasedOperatorConversion(CHARACTER_LENGTH_CONVERSION, "STRLEN")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.CONCAT, "concat")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.EXP, "exp")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.DIVIDE_INTEGER, "div")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.LN, "log")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.LOWER, "lower")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.LOG10, "log10")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.POWER, "pow")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.REPLACE, "replace")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.SQRT, "sqrt")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.UPPER, "upper")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.PI, "pi")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.SIN, "sin")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.COS, "cos")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.TAN, "tan")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.COT, "cot")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.ASIN, "asin")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.ACOS, "acos")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.ATAN, "atan")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.ATAN2, "atan2")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.RADIANS, "toRadians")) - .add(new DirectOperatorConversion(SqlStdOperatorTable.DEGREES, "toDegrees")) - .add(new UnaryPrefixOperatorConversion(SqlStdOperatorTable.NOT, "!")) - .add(new UnaryPrefixOperatorConversion(SqlStdOperatorTable.UNARY_MINUS, "-")) - .add(new UnaryFunctionOperatorConversion(SqlStdOperatorTable.IS_NULL, "isnull")) - .add(new UnaryFunctionOperatorConversion(SqlStdOperatorTable.IS_NOT_NULL, "notnull")) - .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_FALSE, "<= 0")) // Matches Evals.asBoolean - .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_NOT_TRUE, "<= 0")) // Matches Evals.asBoolean - .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_TRUE, "> 0")) // Matches Evals.asBoolean - .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_NOT_FALSE, "> 0")) // Matches Evals.asBoolean - .add(new BinaryOperatorConversion(SqlStdOperatorTable.MULTIPLY, "*")) - .add(new BinaryOperatorConversion(SqlStdOperatorTable.MOD, "%")) - .add(new BinaryOperatorConversion(SqlStdOperatorTable.DIVIDE, "/")) - .add(new BinaryOperatorConversion(SqlStdOperatorTable.PLUS, "+")) - .add(new BinaryOperatorConversion(SqlStdOperatorTable.MINUS, "-")) - .add(new BinaryOperatorConversion(SqlStdOperatorTable.EQUALS, "==")) - .add(new BinaryOperatorConversion(SqlStdOperatorTable.NOT_EQUALS, "!=")) - .add(new BinaryOperatorConversion(SqlStdOperatorTable.GREATER_THAN, ">")) - .add(new BinaryOperatorConversion(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, ">=")) - .add(new BinaryOperatorConversion(SqlStdOperatorTable.LESS_THAN, "<")) - .add(new BinaryOperatorConversion(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, "<=")) - .add(new BinaryOperatorConversion(SqlStdOperatorTable.AND, "&&")) - .add(new BinaryOperatorConversion(SqlStdOperatorTable.OR, "||")) - .add(new RoundOperatorConversion()) - .addAll(TIME_OPERATOR_CONVERSIONS) - .addAll(STRING_OPERATOR_CONVERSIONS) - .addAll(VALUE_COERCION_OPERATOR_CONVERSIONS) - .addAll(ARRAY_OPERATOR_CONVERSIONS) - .addAll(MULTIVALUE_STRING_OPERATOR_CONVERSIONS) - .addAll(REDUCTION_OPERATOR_CONVERSIONS) - .addAll(IPV4ADDRESS_OPERATOR_CONVERSIONS) - .addAll(BITWISE_OPERATOR_CONVERSIONS) - .build(); + .add(new DirectOperatorConversion(SqlStdOperatorTable.ABS, "abs")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.CASE, "case_searched")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.CHAR_LENGTH, "strlen")) + .add(CHARACTER_LENGTH_CONVERSION) + .add(new AliasedOperatorConversion(CHARACTER_LENGTH_CONVERSION, "LENGTH")) + .add(new AliasedOperatorConversion(CHARACTER_LENGTH_CONVERSION, "STRLEN")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.CONCAT, "concat")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.EXP, "exp")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.DIVIDE_INTEGER, "div")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.LN, "log")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.LOWER, "lower")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.LOG10, "log10")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.POWER, "pow")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.REPLACE, "replace")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.SQRT, "sqrt")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.UPPER, "upper")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.PI, "pi")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.SIN, "sin")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.COS, "cos")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.TAN, "tan")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.COT, "cot")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.ASIN, "asin")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.ACOS, "acos")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.ATAN, "atan")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.ATAN2, "atan2")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.RADIANS, "toRadians")) + .add(new DirectOperatorConversion(SqlStdOperatorTable.DEGREES, "toDegrees")) + .add(new UnaryPrefixOperatorConversion(SqlStdOperatorTable.NOT, "!")) + .add(new UnaryPrefixOperatorConversion(SqlStdOperatorTable.UNARY_MINUS, "-")) + .add(new UnaryFunctionOperatorConversion(SqlStdOperatorTable.IS_NULL, "isnull")) + .add(new UnaryFunctionOperatorConversion(SqlStdOperatorTable.IS_NOT_NULL, "notnull")) + .add(new UnarySuffixOperatorConversion( + SqlStdOperatorTable.IS_FALSE, + "<= 0" + )) // Matches Evals.asBoolean + .add(new UnarySuffixOperatorConversion( + SqlStdOperatorTable.IS_NOT_TRUE, + "<= 0" + )) // Matches Evals.asBoolean + .add(new UnarySuffixOperatorConversion( + SqlStdOperatorTable.IS_TRUE, + "> 0" + )) // Matches Evals.asBoolean + .add(new UnarySuffixOperatorConversion( + SqlStdOperatorTable.IS_NOT_FALSE, + "> 0" + )) // Matches Evals.asBoolean + .add(new BinaryOperatorConversion(SqlStdOperatorTable.MULTIPLY, "*")) + .add(new BinaryOperatorConversion(SqlStdOperatorTable.MOD, "%")) + .add(new BinaryOperatorConversion(SqlStdOperatorTable.DIVIDE, "/")) + .add(new BinaryOperatorConversion(SqlStdOperatorTable.PLUS, "+")) + .add(new BinaryOperatorConversion(SqlStdOperatorTable.MINUS, "-")) + .add(new BinaryOperatorConversion(SqlStdOperatorTable.EQUALS, "==")) + .add(new BinaryOperatorConversion(SqlStdOperatorTable.NOT_EQUALS, "!=")) + .add(new BinaryOperatorConversion(SqlStdOperatorTable.GREATER_THAN, ">")) + .add(new BinaryOperatorConversion(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, ">=")) + .add(new BinaryOperatorConversion(SqlStdOperatorTable.LESS_THAN, "<")) + .add(new BinaryOperatorConversion(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, "<=")) + .add(new BinaryOperatorConversion(SqlStdOperatorTable.AND, "&&")) + .add(new BinaryOperatorConversion(SqlStdOperatorTable.OR, "||")) + .add(new RoundOperatorConversion()) + .addAll(TIME_OPERATOR_CONVERSIONS) + .addAll(STRING_OPERATOR_CONVERSIONS) + .addAll(VALUE_COERCION_OPERATOR_CONVERSIONS) + .addAll(ARRAY_OPERATOR_CONVERSIONS) + .addAll(MULTIVALUE_STRING_OPERATOR_CONVERSIONS) + .addAll(REDUCTION_OPERATOR_CONVERSIONS) + .addAll(IPV4ADDRESS_OPERATOR_CONVERSIONS) + .addAll(BITWISE_OPERATOR_CONVERSIONS) + .build(); // Operators that have no conversion, but are handled in the convertlet table, so they still need to exist. private static final Map CONVERTLET_OPERATORS = diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java index 344023e5864..0d7f979d42f 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java @@ -28,6 +28,7 @@ import org.apache.calcite.avatica.remote.TypedValue; import org.apache.calcite.linq4j.QueryProvider; import org.apache.calcite.schema.SchemaPlus; import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Numbers; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.AuthenticationResult; @@ -54,6 +55,7 @@ public class PlannerContext public static final String CTX_SQL_QUERY_ID = "sqlQueryId"; public static final String CTX_SQL_CURRENT_TIMESTAMP = "sqlCurrentTimestamp"; public static final String CTX_SQL_TIME_ZONE = "sqlTimeZone"; + public static final String CTX_SQL_STRINGIFY_ARRAYS = "sqlStringifyArrays"; // This context parameter is an undocumented parameter, used internally, to allow the web console to // apply a limit without having to rewrite the SQL query. @@ -68,6 +70,7 @@ public class PlannerContext private final DateTime localNow; private final Map queryContext; private final String sqlQueryId; + private final boolean stringifyArrays; private final List nativeQueryIds = new CopyOnWriteArrayList<>(); // bindings for dynamic parameters to bind during planning private List parameters = Collections.emptyList(); @@ -83,6 +86,7 @@ public class PlannerContext final ExprMacroTable macroTable, final PlannerConfig plannerConfig, final DateTime localNow, + final boolean stringifyArrays, final Map queryContext ) { @@ -91,6 +95,7 @@ public class PlannerContext this.plannerConfig = Preconditions.checkNotNull(plannerConfig, "plannerConfig"); this.queryContext = queryContext != null ? new HashMap<>(queryContext) : new HashMap<>(); this.localNow = Preconditions.checkNotNull(localNow, "localNow"); + this.stringifyArrays = stringifyArrays; String sqlQueryId = (String) this.queryContext.get(CTX_SQL_QUERY_ID); // special handling for DruidViewMacro, normal client will allocate sqlid in SqlLifecyle @@ -109,8 +114,10 @@ public class PlannerContext { final DateTime utcNow; final DateTimeZone timeZone; + final boolean stringifyArrays; if (queryContext != null) { + final Object stringifyParam = queryContext.get(CTX_SQL_STRINGIFY_ARRAYS); final Object tsParam = queryContext.get(CTX_SQL_CURRENT_TIMESTAMP); final Object tzParam = queryContext.get(CTX_SQL_TIME_ZONE); @@ -125,9 +132,16 @@ public class PlannerContext } else { timeZone = plannerConfig.getSqlTimeZone(); } + + if (stringifyParam != null) { + stringifyArrays = Numbers.parseBoolean(stringifyParam); + } else { + stringifyArrays = true; + } } else { utcNow = new DateTime(DateTimeZone.UTC); timeZone = plannerConfig.getSqlTimeZone(); + stringifyArrays = true; } return new PlannerContext( @@ -135,6 +149,7 @@ public class PlannerContext macroTable, plannerConfig.withOverrides(queryContext), utcNow.withZone(timeZone), + stringifyArrays, queryContext ); } @@ -169,6 +184,11 @@ public class PlannerContext return queryContext; } + public boolean isStringifyArrays() + { + return stringifyArrays; + } + public List getParameters() { return parameters; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/QueryMaker.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/QueryMaker.java index 5def03293be..0a283209281 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/QueryMaker.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/QueryMaker.java @@ -53,6 +53,8 @@ import org.joda.time.DateTime; import org.joda.time.Interval; import java.io.IOException; +import java.sql.Array; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.UUID; @@ -211,9 +213,11 @@ public class QueryMaker } else if (sqlType == SqlTypeName.DATE) { return ColumnMetaData.Rep.of(Integer.class); } else if (sqlType == SqlTypeName.INTEGER) { - return ColumnMetaData.Rep.of(Integer.class); + // use Number.class for exact numeric types since JSON transport might switch longs to integers + return ColumnMetaData.Rep.of(Number.class); } else if (sqlType == SqlTypeName.BIGINT) { - return ColumnMetaData.Rep.of(Long.class); + // use Number.class for exact numeric types since JSON transport might switch longs to integers + return ColumnMetaData.Rep.of(Number.class); } else if (sqlType == SqlTypeName.FLOAT) { return ColumnMetaData.Rep.of(Float.class); } else if (sqlType == SqlTypeName.DOUBLE || sqlType == SqlTypeName.DECIMAL) { @@ -222,6 +226,8 @@ public class QueryMaker return ColumnMetaData.Rep.of(Boolean.class); } else if (sqlType == SqlTypeName.OTHER) { return ColumnMetaData.Rep.of(Object.class); + } else if (sqlType == SqlTypeName.ARRAY) { + return ColumnMetaData.Rep.of(Array.class); } else { throw new ISE("No rep for SQL type[%s]", sqlType); } @@ -309,16 +315,33 @@ public class QueryMaker coercedValue = value.getClass().getName(); } } else if (sqlType == SqlTypeName.ARRAY) { - if (value instanceof String) { - coercedValue = NullHandling.nullToEmptyIfNeeded((String) value); - } else if (value instanceof NlsString) { - coercedValue = ((NlsString) value).getValue(); - } else { - try { - coercedValue = jsonMapper.writeValueAsString(value); + if (plannerContext.isStringifyArrays()) { + if (value instanceof String) { + coercedValue = NullHandling.nullToEmptyIfNeeded((String) value); + } else if (value instanceof NlsString) { + coercedValue = ((NlsString) value).getValue(); + } else { + try { + coercedValue = jsonMapper.writeValueAsString(value); + } + catch (IOException e) { + throw new RuntimeException(e); + } } - catch (IOException e) { - throw new RuntimeException(e); + } else { + // the protobuf jdbc handler prefers lists (it actually can't handle java arrays as sql arrays, only java lists) + // the json handler could handle this just fine, but it handles lists as sql arrays as well so just convert + // here if needed + if (value instanceof List) { + coercedValue = value; + } else if (value instanceof String[]) { + coercedValue = Arrays.asList((String[]) value); + } else if (value instanceof Long[]) { + coercedValue = Arrays.asList((Long[]) value); + } else if (value instanceof Double[]) { + coercedValue = Arrays.asList((Double[]) value); + } else { + throw new ISE("Cannot coerce[%s] to %s", value.getClass().getName(), sqlType); } } } else { diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java index 7a3a0340fa3..ef72cbd7e68 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java @@ -81,6 +81,7 @@ import org.junit.rules.TemporaryFolder; import java.io.IOException; import java.net.InetSocketAddress; +import java.sql.Array; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.Date; @@ -337,7 +338,7 @@ public abstract class DruidAvaticaHandlerTest extends CalciteTestBase ImmutableList.of( ImmutableMap.of( "PLAN", - StringUtils.format("DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"%s\",\"sqlTimeZone\":\"America/Los_Angeles\"}}], signature=[{a0:LONG}])\n", + StringUtils.format("DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"%s\",\"sqlStringifyArrays\":false,\"sqlTimeZone\":\"America/Los_Angeles\"}}], signature=[{a0:LONG}])\n", DUMMY_SQL_QUERY_ID ), "RESOURCES", @@ -1316,6 +1317,33 @@ public abstract class DruidAvaticaHandlerTest extends CalciteTestBase ); } + + @Test + public void testArrayStuffs() throws Exception + { + PreparedStatement statement = client.prepareStatement( + "SELECT ARRAY_AGG(dim2) AS arr1, ARRAY_AGG(l1) AS arr2, ARRAY_AGG(d1) AS arr3, ARRAY_AGG(f1) AS arr4 FROM druid.numfoo" + ); + final ResultSet resultSet = statement.executeQuery(); + final List> rows = getRows(resultSet); + Assert.assertEquals(1, rows.size()); + Assert.assertTrue(rows.get(0).containsKey("arr1")); + Assert.assertTrue(rows.get(0).containsKey("arr2")); + Assert.assertTrue(rows.get(0).containsKey("arr3")); + Assert.assertTrue(rows.get(0).containsKey("arr4")); + if (NullHandling.sqlCompatible()) { + Assert.assertArrayEquals(new Object[]{"a", null, "", "a", "abc", null}, (Object[]) rows.get(0).get("arr1")); + Assert.assertArrayEquals(new Object[]{7L, 325323L, 0L, null, null, null}, (Object[]) rows.get(0).get("arr2")); + Assert.assertArrayEquals(new Object[]{1.0, 1.7, 0.0, null, null, null}, (Object[]) rows.get(0).get("arr3")); + Assert.assertArrayEquals(new Object[]{1.0f, 0.1f, 0.0f, null, null, null}, (Object[]) rows.get(0).get("arr4")); + } else { + Assert.assertArrayEquals(new Object[]{"a", null, null, "a", "abc", null}, (Object[]) rows.get(0).get("arr1")); + Assert.assertArrayEquals(new Object[]{7L, 325323L, 0L, 0L, 0L, 0L}, (Object[]) rows.get(0).get("arr2")); + Assert.assertArrayEquals(new Object[]{1.0, 1.7, 0.0, 0.0, 0.0, 0.0}, (Object[]) rows.get(0).get("arr3")); + Assert.assertArrayEquals(new Object[]{1.0f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f}, (Object[]) rows.get(0).get("arr4")); + } + } + protected abstract String getJdbcConnectionString(int port); protected abstract AbstractAvaticaHandler getAvaticaHandler(DruidMeta druidMeta); @@ -1335,7 +1363,12 @@ public abstract class DruidAvaticaHandlerTest extends CalciteTestBase final Map row = new HashMap<>(); for (int i = 0; i < metaData.getColumnCount(); i++) { if (returnKeys == null || returnKeys.contains(metaData.getColumnLabel(i + 1))) { - row.put(metaData.getColumnLabel(i + 1), resultSet.getObject(i + 1)); + Object result = resultSet.getObject(i + 1); + if (result instanceof Array) { + row.put(metaData.getColumnLabel(i + 1), ((Array) result).getArray()); + } else { + row.put(metaData.getColumnLabel(i + 1), result); + } } } rows.add(row); diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java index f58a3fb8807..44002866a32 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java @@ -124,7 +124,7 @@ public class DruidStatementTest extends CalciteTestBase Assert.assertEquals( Lists.newArrayList( Lists.newArrayList("__time", "TIMESTAMP", "java.lang.Long"), - Lists.newArrayList("cnt", "BIGINT", "java.lang.Long"), + Lists.newArrayList("cnt", "BIGINT", "java.lang.Number"), Lists.newArrayList("dim1", "VARCHAR", "java.lang.String"), Lists.newArrayList("dim2", "VARCHAR", "java.lang.String"), Lists.newArrayList("dim3", "VARCHAR", "java.lang.String"), diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index 645f9b9e0b9..adc32f613cd 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -112,6 +112,8 @@ public class BaseCalciteQueryTest extends CalciteTestBase public static Long NULL_LONG; public static final String HLLC_STRING = VersionOneHyperLogLogCollector.class.getName(); + final boolean useDefault = NullHandling.replaceWithDefault(); + @BeforeClass public static void setupNullValues() { @@ -175,6 +177,10 @@ public class BaseCalciteQueryTest extends CalciteTestBase .put(QueryContexts.MAX_SCATTER_GATHER_BYTES_KEY, Long.MAX_VALUE); public static final Map QUERY_CONTEXT_DEFAULT = DEFAULT_QUERY_CONTEXT_BUILDER.build(); + public static final Map QUERY_CONTEXT_NO_STRINGIFY_ARRAY = + DEFAULT_QUERY_CONTEXT_BUILDER.put(PlannerContext.CTX_SQL_STRINGIFY_ARRAYS, false) + .build(); + public static final Map QUERY_CONTEXT_DONT_SKIP_EMPTY_BUCKETS = ImmutableMap.of( PlannerContext.CTX_SQL_QUERY_ID, DUMMY_SQL_ID, PlannerContext.CTX_SQL_CURRENT_TIMESTAMP, "2000-01-01T00:00:00Z", diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java new file mode 100644 index 00000000000..aded4f8c283 --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -0,0 +1,1763 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import junitparams.JUnitParamsRunner; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.HumanReadableBytes; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.QueryDataSource; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.ExpressionLambdaAggregatorFactory; +import org.apache.druid.query.aggregation.FilteredAggregatorFactory; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.dimension.DefaultDimensionSpec; +import org.apache.druid.query.expression.TestExprMacroTable; +import org.apache.druid.query.filter.AndDimFilter; +import org.apache.druid.query.filter.ExpressionDimFilter; +import org.apache.druid.query.filter.InDimFilter; +import org.apache.druid.query.filter.SelectorDimFilter; +import org.apache.druid.query.groupby.GroupByQuery; +import org.apache.druid.query.groupby.orderby.DefaultLimitSpec; +import org.apache.druid.query.groupby.orderby.NoopLimitSpec; +import org.apache.druid.query.groupby.orderby.OrderByColumnSpec; +import org.apache.druid.query.ordering.StringComparators; +import org.apache.druid.query.scan.ScanQuery; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.join.JoinType; +import org.apache.druid.sql.calcite.filtration.Filtration; +import org.apache.druid.sql.calcite.util.CalciteTests; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.List; + +/** + * Tests for array functions and array types + */ +@RunWith(JUnitParamsRunner.class) +public class CalciteArraysQueryTest extends BaseCalciteQueryTest +{ + // test some query stuffs, sort of limited since no native array column types so either need to use constructor or + // array aggregator + @Test + public void testSelectConstantArrayExpressionFromTable() throws Exception + { + testQuery( + "SELECT ARRAY[1,2] as arr, dim1 FROM foo LIMIT 1", + ImmutableList.of( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .virtualColumns(expressionVirtualColumn("v0", "array(1,2)", ValueType.STRING)) + .columns("dim1", "v0") + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .limit(1) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{"[1,2]", ""} + ) + ); + } + + @Test + public void testGroupByArrayFromCase() throws Exception + { + cannotVectorize(); + testQuery( + "SELECT CASE WHEN dim4 = 'a' THEN ARRAY['foo','bar','baz'] END as mv_value, count(1) from numfoo GROUP BY 1", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setVirtualColumns(expressionVirtualColumn( + "v0", + "case_searched((\"dim4\" == 'a'),array('foo','bar','baz'),null)", + ValueType.STRING + )) + .setDimensions(new DefaultDimensionSpec("v0", "_d0")) + .setGranularity(Granularities.ALL) + .setAggregatorSpecs(new CountAggregatorFactory("a0")) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{null, 3L}, + new Object[]{"bar", 3L}, + new Object[]{"baz", 3L}, + new Object[]{"foo", 3L} + ) + ); + } + + @Test + public void testSelectNonConstantArrayExpressionFromTable() throws Exception + { + testQuery( + "SELECT ARRAY[CONCAT(dim1, 'word'),'up'] as arr, dim1 FROM foo LIMIT 5", + ImmutableList.of( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .virtualColumns(expressionVirtualColumn("v0", "array(concat(\"dim1\",'word'),'up')", ValueType.STRING)) + .columns("dim1", "v0") + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .limit(5) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{"[\"word\",\"up\"]", ""}, + new Object[]{"[\"10.1word\",\"up\"]", "10.1"}, + new Object[]{"[\"2word\",\"up\"]", "2"}, + new Object[]{"[\"1word\",\"up\"]", "1"}, + new Object[]{"[\"defword\",\"up\"]", "def"} + ) + ); + } + + @Test + public void testSelectNonConstantArrayExpressionFromTableFailForMultival() throws Exception + { + // without expression output type inference to prevent this, the automatic translation will try to turn this into + // + // `map((dim3) -> array(concat(dim3,'word'),'up'), dim3)` + // + // This error message will get better in the future. The error without translation would be: + // + // org.apache.druid.java.util.common.RE: Unhandled array constructor element type [STRING_ARRAY] + + expectedException.expect(RuntimeException.class); + expectedException.expectMessage("Unhandled map function output type [STRING_ARRAY]"); + testQuery( + "SELECT ARRAY[CONCAT(dim3, 'word'),'up'] as arr, dim1 FROM foo LIMIT 5", + ImmutableList.of(), + ImmutableList.of() + ); + } + + @Test + public void testSomeArrayFunctionsWithScanQuery() throws Exception + { + // array constructor turns decimals into ints for some reason, this needs fixed in the future + // also, yes these outputs are strange sometimes, arrays are in a partial state of existence so end up a bit + // stringy for now this is because virtual column selectors are coercing values back to stringish so that + // multi-valued string dimensions can be grouped on. + List expectedResults; + if (useDefault) { + expectedResults = ImmutableList.of( + new Object[]{ + "", + "a", + "[\"a\",\"b\"]", + 7L, + 0L, + 1.0, + 0.0, + "[\"a\",\"b\",\"c\"]", + "[1,2,3]", + "[1,2,4]", + "[\"a\",\"b\",\"foo\"]", + "[\"foo\",\"a\"]", + "[1,2,7]", + "[0,1,2]", + "[1,2,1]", + "[0,1,2]", + "[\"a\",\"a\",\"b\"]", + "[7,0]", + "[1.0,0.0]", + "7", + "1.0", + "7", + "1.0" + } + ); + } else { + expectedResults = ImmutableList.of( + new Object[]{ + "", + "a", + "[\"a\",\"b\"]", + 7L, + null, + 1.0, + null, + "[\"a\",\"b\",\"c\"]", + "[1,2,3]", + "[1,2,4]", + "[\"a\",\"b\",\"foo\"]", + "[\"foo\",\"a\"]", + "[1,2,7]", + "[null,1,2]", + "[1,2,1]", + "[null,1,2]", + "[\"a\",\"a\",\"b\"]", + "[7,null]", + "[1.0,null]", + "7", + "1.0", + "7", + "1.0" + } + ); + } + testQuery( + "SELECT" + + " dim1," + + " dim2," + + " dim3," + + " l1," + + " l2," + + " d1," + + " d2," + + " ARRAY['a', 'b', 'c']," + + " ARRAY[1,2,3]," + + " ARRAY[1.9, 2.2, 4.3]," + + " ARRAY_APPEND(dim3, 'foo')," + + " ARRAY_PREPEND('foo', ARRAY[dim2])," + + " ARRAY_APPEND(ARRAY[1,2], l1)," + + " ARRAY_PREPEND(l2, ARRAY[1,2])," + + " ARRAY_APPEND(ARRAY[1.2,2.2], d1)," + + " ARRAY_PREPEND(d2, ARRAY[1.1,2.2])," + + " ARRAY_CONCAT(dim2,dim3)," + + " ARRAY_CONCAT(ARRAY[l1],ARRAY[l2])," + + " ARRAY_CONCAT(ARRAY[d1],ARRAY[d2])," + + " ARRAY_OFFSET(ARRAY[l1],0)," + + " ARRAY_OFFSET(ARRAY[d1],0)," + + " ARRAY_ORDINAL(ARRAY[l1],1)," + + " ARRAY_ORDINAL(ARRAY[d1],1)" + + " FROM druid.numfoo" + + " LIMIT 1", + ImmutableList.of( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .virtualColumns( + // these report as strings even though they are not, someday this will not be so + expressionVirtualColumn("v0", "array('a','b','c')", ValueType.STRING), + expressionVirtualColumn("v1", "array(1,2,3)", ValueType.STRING), + expressionVirtualColumn("v10", "array_concat(array(\"l1\"),array(\"l2\"))", ValueType.STRING), + expressionVirtualColumn("v11", "array_concat(array(\"d1\"),array(\"d2\"))", ValueType.STRING), + expressionVirtualColumn("v12", "array_offset(array(\"l1\"),0)", ValueType.STRING), + expressionVirtualColumn("v13", "array_offset(array(\"d1\"),0)", ValueType.STRING), + expressionVirtualColumn("v14", "array_ordinal(array(\"l1\"),1)", ValueType.STRING), + expressionVirtualColumn("v15", "array_ordinal(array(\"d1\"),1)", ValueType.STRING), + expressionVirtualColumn("v2", "array(1,2,4)", ValueType.STRING), + expressionVirtualColumn("v3", "array_append(\"dim3\",'foo')", ValueType.STRING), + expressionVirtualColumn("v4", "array_prepend('foo',array(\"dim2\"))", ValueType.STRING), + expressionVirtualColumn("v5", "array_append(array(1,2),\"l1\")", ValueType.STRING), + expressionVirtualColumn("v6", "array_prepend(\"l2\",array(1,2))", ValueType.STRING), + expressionVirtualColumn("v7", "array_append(array(1,2),\"d1\")", ValueType.STRING), + expressionVirtualColumn("v8", "array_prepend(\"d2\",array(1,2))", ValueType.STRING), + expressionVirtualColumn("v9", "array_concat(\"dim2\",\"dim3\")", ValueType.STRING) + ) + .columns( + "d1", + "d2", + "dim1", + "dim2", + "dim3", + "l1", + "l2", + "v0", + "v1", + "v10", + "v11", + "v12", + "v13", + "v14", + "v15", + "v2", + "v3", + "v4", + "v5", + "v6", + "v7", + "v8", + "v9" + ) + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .limit(1) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + expectedResults + ); + } + + @Test + public void testSomeArrayFunctionsWithScanQueryNoStringify() throws Exception + { + // when not stringifying arrays, some things are still stringified, because they are inferred to be typed as strings + // the planner context which controls stringification of arrays does not apply to multi-valued string columns, + // which will still always be stringified to ultimately adhere to the varchar type + // as array support increases in the engine this will likely change since using explict array functions should + // probably kick it into an array + List expectedResults; + if (useDefault) { + expectedResults = ImmutableList.of( + new Object[]{ + "", + "a", + "[\"a\",\"b\"]", + Arrays.asList("a", "b", "c"), + Arrays.asList(1L, 2L, 3L), + Arrays.asList(1L, 2L, 4L), + "[\"a\",\"b\",\"foo\"]", + Arrays.asList("foo", "a"), + Arrays.asList(1L, 2L, 7L), + Arrays.asList(0L, 1L, 2L), + Arrays.asList(1L, 2L, 1L), + Arrays.asList(0L, 1L, 2L), + "[\"a\",\"a\",\"b\"]", + Arrays.asList(7L, 0L), + Arrays.asList(1.0, 0.0) + } + ); + } else { + expectedResults = ImmutableList.of( + new Object[]{ + "", + "a", + "[\"a\",\"b\"]", + Arrays.asList("a", "b", "c"), + Arrays.asList(1L, 2L, 3L), + Arrays.asList(1L, 2L, 4L), + "[\"a\",\"b\",\"foo\"]", + Arrays.asList("foo", "a"), + Arrays.asList(1L, 2L, 7L), + Arrays.asList(null, 1L, 2L), + Arrays.asList(1L, 2L, 1L), + Arrays.asList(null, 1L, 2L), + "[\"a\",\"a\",\"b\"]", + Arrays.asList(7L, null), + Arrays.asList(1.0, null) + } + ); + } + testQuery( + "SELECT" + + " dim1," + + " dim2," + + " dim3," + + " ARRAY['a', 'b', 'c']," + + " ARRAY[1,2,3]," + + " ARRAY[1.9, 2.2, 4.3]," + + " ARRAY_APPEND(dim3, 'foo')," + + " ARRAY_PREPEND('foo', ARRAY[dim2])," + + " ARRAY_APPEND(ARRAY[1,2], l1)," + + " ARRAY_PREPEND(l2, ARRAY[1,2])," + + " ARRAY_APPEND(ARRAY[1.2,2.2], d1)," + + " ARRAY_PREPEND(d2, ARRAY[1.1,2.2])," + + " ARRAY_CONCAT(dim2,dim3)," + + " ARRAY_CONCAT(ARRAY[l1],ARRAY[l2])," + + " ARRAY_CONCAT(ARRAY[d1],ARRAY[d2])" + + " FROM druid.numfoo" + + " LIMIT 1", + QUERY_CONTEXT_NO_STRINGIFY_ARRAY, + ImmutableList.of( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .virtualColumns( + // these report as strings even though they are not, someday this will not be so + expressionVirtualColumn("v0", "array('a','b','c')", ValueType.STRING), + expressionVirtualColumn("v1", "array(1,2,3)", ValueType.STRING), + expressionVirtualColumn("v10", "array_concat(array(\"l1\"),array(\"l2\"))", ValueType.STRING), + expressionVirtualColumn("v11", "array_concat(array(\"d1\"),array(\"d2\"))", ValueType.STRING), + expressionVirtualColumn("v2", "array(1,2,4)", ValueType.STRING), + expressionVirtualColumn("v3", "array_append(\"dim3\",'foo')", ValueType.STRING), + expressionVirtualColumn("v4", "array_prepend('foo',array(\"dim2\"))", ValueType.STRING), + expressionVirtualColumn("v5", "array_append(array(1,2),\"l1\")", ValueType.STRING), + expressionVirtualColumn("v6", "array_prepend(\"l2\",array(1,2))", ValueType.STRING), + expressionVirtualColumn("v7", "array_append(array(1,2),\"d1\")", ValueType.STRING), + expressionVirtualColumn("v8", "array_prepend(\"d2\",array(1,2))", ValueType.STRING), + expressionVirtualColumn("v9", "array_concat(\"dim2\",\"dim3\")", ValueType.STRING) + ) + .columns( + "dim1", + "dim2", + "dim3", + "v0", + "v1", + "v10", + "v11", + "v2", + "v3", + "v4", + "v5", + "v6", + "v7", + "v8", + "v9" + ) + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .limit(1) + .context(QUERY_CONTEXT_NO_STRINGIFY_ARRAY) + .build() + ), + expectedResults + ); + } + + @Test + public void testArrayOverlapFilter() throws Exception + { + testQuery( + "SELECT dim3 FROM druid.numfoo WHERE ARRAY_OVERLAP(dim3, ARRAY['a','b']) LIMIT 5", + ImmutableList.of( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .filters(new InDimFilter("dim3", ImmutableList.of("a", "b"), null)) + .columns("dim3") + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .limit(5) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{"[\"a\",\"b\"]"}, + new Object[]{"[\"b\",\"c\"]"} + ) + ); + } + + @Test + public void testArrayOverlapFilterNonLiteral() throws Exception + { + testQuery( + "SELECT dim3 FROM druid.numfoo WHERE ARRAY_OVERLAP(dim3, ARRAY[dim2]) LIMIT 5", + ImmutableList.of( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .filters(expressionFilter("array_overlap(\"dim3\",array(\"dim2\"))")) + .columns("dim3") + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .limit(5) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{"[\"a\",\"b\"]"}, + new Object[]{useDefault ? "" : null} + ) + ); + } + + @Test + public void testArrayContainsFilter() throws Exception + { + testQuery( + "SELECT dim3 FROM druid.numfoo WHERE ARRAY_CONTAINS(dim3, ARRAY['a','b']) LIMIT 5", + ImmutableList.of( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .filters( + new AndDimFilter( + new SelectorDimFilter("dim3", "a", null), + new SelectorDimFilter("dim3", "b", null) + ) + ) + .columns("dim3") + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .limit(5) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{"[\"a\",\"b\"]"} + ) + ); + } + + @Test + public void testArrayContainsArrayOfOneElement() throws Exception + { + testQuery( + "SELECT dim3 FROM druid.numfoo WHERE ARRAY_CONTAINS(dim3, ARRAY['a']) LIMIT 5", + ImmutableList.of( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .filters(new SelectorDimFilter("dim3", "a", null)) + .columns("dim3") + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .limit(5) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{"[\"a\",\"b\"]"} + ) + ); + } + + @Test + public void testArrayContainsArrayOfNonLiteral() throws Exception + { + testQuery( + "SELECT dim3 FROM druid.numfoo WHERE ARRAY_CONTAINS(dim3, ARRAY[dim2]) LIMIT 5", + ImmutableList.of( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .filters(expressionFilter("array_contains(\"dim3\",array(\"dim2\"))")) + .columns("dim3") + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .limit(5) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{"[\"a\",\"b\"]"}, + new Object[]{useDefault ? "" : null} + ) + ); + } + + @Test + public void testArraySlice() throws Exception + { + testQuery( + "SELECT ARRAY_SLICE(dim3, 1) FROM druid.numfoo", + ImmutableList.of( + new Druids.ScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .virtualColumns(expressionVirtualColumn("v0", "array_slice(\"dim3\",1)", ValueType.STRING)) + .columns(ImmutableList.of("v0")) + .context(QUERY_CONTEXT_DEFAULT) + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .legacy(false) + .build() + ), + ImmutableList.of( + new Object[]{"[\"b\"]"}, + new Object[]{"[\"c\"]"}, + new Object[]{"[]"}, + new Object[]{"[]"}, + new Object[]{"[]"}, + new Object[]{"[]"} + ) + ); + } + + @Test + public void testArrayLength() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + testQuery( + "SELECT dim1, ARRAY_LENGTH(dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1, 2 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn("v0", "array_length(\"dim3\")", ValueType.LONG)) + .setDimensions( + dimensions( + new DefaultDimensionSpec("dim1", "_d0", ValueType.STRING), + new DefaultDimensionSpec("v0", "_d1", ValueType.LONG) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "_d1", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{"", 2, 1L}, + new Object[]{"10.1", 2, 1L}, + new Object[]{"1", 1, 1L}, + new Object[]{"2", 1, 1L}, + new Object[]{"abc", 1, 1L}, + new Object[]{"def", 1, 1L} + ) + ); + } + + @Test + public void testArrayAppend() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + ImmutableList results; + if (useDefault) { + results = ImmutableList.of( + new Object[]{"foo", 6L}, + new Object[]{"", 3L}, + new Object[]{"b", 2L}, + new Object[]{"a", 1L}, + new Object[]{"c", 1L}, + new Object[]{"d", 1L} + ); + } else { + results = ImmutableList.of( + new Object[]{"foo", 6L}, + new Object[]{null, 2L}, + new Object[]{"b", 2L}, + new Object[]{"", 1L}, + new Object[]{"a", 1L}, + new Object[]{"c", 1L}, + new Object[]{"d", 1L} + ); + } + testQuery( + "SELECT ARRAY_APPEND(dim3, 'foo'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn( + "v0", + "array_append(\"dim3\",'foo')", + ValueType.STRING + )) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + results + ); + } + + @Test + public void testArrayPrepend() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + ImmutableList results; + if (useDefault) { + results = ImmutableList.of( + new Object[]{"foo", 6L}, + new Object[]{"", 3L}, + new Object[]{"b", 2L}, + new Object[]{"a", 1L}, + new Object[]{"c", 1L}, + new Object[]{"d", 1L} + ); + } else { + results = ImmutableList.of( + new Object[]{"foo", 6L}, + new Object[]{null, 2L}, + new Object[]{"b", 2L}, + new Object[]{"", 1L}, + new Object[]{"a", 1L}, + new Object[]{"c", 1L}, + new Object[]{"d", 1L} + ); + } + testQuery( + "SELECT ARRAY_PREPEND('foo', dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn( + "v0", + "array_prepend('foo',\"dim3\")", + ValueType.STRING + )) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + results + ); + } + + @Test + public void testArrayPrependAppend() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + ImmutableList results; + if (useDefault) { + results = ImmutableList.of( + new Object[]{"foo,null", "null,foo", 3L}, + new Object[]{"foo,a,b", "a,b,foo", 1L}, + new Object[]{"foo,b,c", "b,c,foo", 1L}, + new Object[]{"foo,d", "d,foo", 1L} + ); + } else { + results = ImmutableList.of( + new Object[]{"foo,null", "null,foo", 2L}, + new Object[]{"foo,", ",foo", 1L}, + new Object[]{"foo,a,b", "a,b,foo", 1L}, + new Object[]{"foo,b,c", "b,c,foo", 1L}, + new Object[]{"foo,d", "d,foo", 1L} + ); + } + testQuery( + "SELECT ARRAY_TO_STRING(ARRAY_PREPEND('foo', dim3), ','), ARRAY_TO_STRING(ARRAY_APPEND(dim3, 'foo'), ','), SUM(cnt) FROM druid.numfoo GROUP BY 1,2 ORDER BY 3 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns( + expressionVirtualColumn( + "v0", + "array_to_string(array_prepend('foo',\"dim3\"),',')", + ValueType.STRING + ), + expressionVirtualColumn( + "v1", + "array_to_string(array_append(\"dim3\",'foo'),',')", + ValueType.STRING + ) + ) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING), + new DefaultDimensionSpec("v1", "_d1", ValueType.STRING) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + results + ); + } + + @Test + public void testArrayConcat() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + ImmutableList results; + if (useDefault) { + results = ImmutableList.of( + new Object[]{"", 6L}, + new Object[]{"b", 4L}, + new Object[]{"a", 2L}, + new Object[]{"c", 2L}, + new Object[]{"d", 2L} + ); + } else { + results = ImmutableList.of( + new Object[]{null, 4L}, + new Object[]{"b", 4L}, + new Object[]{"", 2L}, + new Object[]{"a", 2L}, + new Object[]{"c", 2L}, + new Object[]{"d", 2L} + ); + } + testQuery( + "SELECT ARRAY_CONCAT(dim3, dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn( + "v0", + "array_concat(\"dim3\",\"dim3\")", + ValueType.STRING + )) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + results + ); + } + + @Test + public void testArrayOffset() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + testQuery( + "SELECT ARRAY_OFFSET(dim3, 1), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn("v0", "array_offset(\"dim3\",1)", ValueType.STRING)) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{NullHandling.defaultStringValue(), 4L}, + new Object[]{"b", 1L}, + new Object[]{"c", 1L} + ) + ); + } + + @Test + public void testArrayOrdinal() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + testQuery( + "SELECT ARRAY_ORDINAL(dim3, 2), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn("v0", "array_ordinal(\"dim3\",2)", ValueType.STRING)) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{NullHandling.defaultStringValue(), 4L}, + new Object[]{"b", 1L}, + new Object[]{"c", 1L} + ) + ); + } + + @Test + public void testArrayOffsetOf() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + testQuery( + "SELECT ARRAY_OFFSET_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn( + "v0", + "array_offset_of(\"dim3\",'b')", + ValueType.LONG + )) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.LONG) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{useDefault ? -1 : null, 4L}, + new Object[]{0, 1L}, + new Object[]{1, 1L} + ) + ); + } + + @Test + public void testArrayOrdinalOf() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + testQuery( + "SELECT ARRAY_ORDINAL_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn( + "v0", + "array_ordinal_of(\"dim3\",'b')", + ValueType.LONG + )) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.LONG) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{useDefault ? -1 : null, 4L}, + new Object[]{1, 1L}, + new Object[]{2, 1L} + ) + ); + } + + @Test + public void testArrayToString() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + ImmutableList results; + if (useDefault) { + results = ImmutableList.of( + new Object[]{"", 3L}, + new Object[]{"a,b", 1L}, + new Object[]{"b,c", 1L}, + new Object[]{"d", 1L} + ); + } else { + results = ImmutableList.of( + new Object[]{null, 2L}, + new Object[]{"", 1L}, + new Object[]{"a,b", 1L}, + new Object[]{"b,c", 1L}, + new Object[]{"d", 1L} + ); + } + testQuery( + "SELECT ARRAY_TO_STRING(dim3, ','), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn( + "v0", + "array_to_string(\"dim3\",',')", + ValueType.STRING + )) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + results + ); + } + + @Test + public void testArrayToStringToMultiValueString() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + ImmutableList results; + if (useDefault) { + results = ImmutableList.of( + new Object[]{"d", 7L}, + new Object[]{null, 3L}, + new Object[]{"b", 2L}, + new Object[]{"a", 1L}, + new Object[]{"c", 1L} + ); + } else { + results = ImmutableList.of( + new Object[]{"d", 5L}, + new Object[]{null, 2L}, + new Object[]{"b", 2L}, + new Object[]{"", 1L}, + new Object[]{"a", 1L}, + new Object[]{"c", 1L} + ); + } + testQuery( + "SELECT STRING_TO_ARRAY(CONCAT(ARRAY_TO_STRING(dim3, ','), ',d'), ','), SUM(cnt) FROM druid.numfoo WHERE ARRAY_LENGTH(dim3) > 0 GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns( + expressionVirtualColumn("v0", "array_length(\"dim3\")", ValueType.LONG), + expressionVirtualColumn( + "v1", + "string_to_array(concat(array_to_string(\"dim3\",','),',d'),',')", + ValueType.STRING + ) + ) + .setDimFilter(bound("v0", "0", null, true, false, null, StringComparators.NUMERIC)) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v1", "_d0", ValueType.STRING) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + results + ); + } + + @Test + public void testArrayAgg() throws Exception + { + cannotVectorize(); + testQuery( + "SELECT ARRAY_AGG(dim1), ARRAY_AGG(DISTINCT dim1), ARRAY_AGG(DISTINCT dim1) FILTER(WHERE dim1 = 'shazbot') FROM foo WHERE dim1 is not null", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(not(selector("dim1", null, null))) + .aggregators( + aggregators( + new ExpressionLambdaAggregatorFactory( + "a0", + ImmutableSet.of("dim1"), + "__acc", + "[]", + "[]", + "array_append(\"__acc\", \"dim1\")", + "array_concat(\"__acc\", \"a0\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + new ExpressionLambdaAggregatorFactory( + "a1", + ImmutableSet.of("dim1"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"dim1\")", + "array_set_add_all(\"__acc\", \"a1\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a2", + ImmutableSet.of("dim1"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"dim1\")", + "array_set_add_all(\"__acc\", \"a2\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + selector("dim1", "shazbot", null) + ) + ) + ) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + useDefault + ? new Object[]{"[\"10.1\",\"2\",\"1\",\"def\",\"abc\"]", "[\"1\",\"2\",\"abc\",\"def\",\"10.1\"]", null} + : new Object[]{ + "[\"\",\"10.1\",\"2\",\"1\",\"def\",\"abc\"]", + "[\"\",\"1\",\"2\",\"abc\",\"def\",\"10.1\"]", + null + } + ) + ); + } + + @Test + public void testArrayAggMultiValue() throws Exception + { + cannotVectorize(); + testQuery( + "SELECT ARRAY_AGG(dim3), ARRAY_AGG(DISTINCT dim3) FROM foo", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .aggregators( + aggregators( + new ExpressionLambdaAggregatorFactory( + "a0", + ImmutableSet.of("dim3"), + "__acc", + "[]", + "[]", + "array_append(\"__acc\", \"dim3\")", + "array_concat(\"__acc\", \"a0\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + new ExpressionLambdaAggregatorFactory( + "a1", + ImmutableSet.of("dim3"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"dim3\")", + "array_set_add_all(\"__acc\", \"a1\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ) + ) + ) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + useDefault + ? new Object[]{"[\"a\",\"b\",\"b\",\"c\",\"d\",null,null,null]", "[null,\"a\",\"b\",\"c\",\"d\"]"} + : new Object[]{"[\"a\",\"b\",\"b\",\"c\",\"d\",\"\",null,null]", "[\"\",null,\"a\",\"b\",\"c\",\"d\"]"} + ) + ); + } + + @Test + public void testArrayAggNumeric() throws Exception + { + cannotVectorize(); + testQuery( + "SELECT ARRAY_AGG(l1), ARRAY_AGG(DISTINCT l1), ARRAY_AGG(d1), ARRAY_AGG(DISTINCT d1), ARRAY_AGG(f1), ARRAY_AGG(DISTINCT f1) FROM numfoo", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .aggregators( + aggregators( + new ExpressionLambdaAggregatorFactory( + "a0", + ImmutableSet.of("l1"), + "__acc", + "[]", + "[]", + "array_append(\"__acc\", \"l1\")", + "array_concat(\"__acc\", \"a0\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + new ExpressionLambdaAggregatorFactory( + "a1", + ImmutableSet.of("l1"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"l1\")", + "array_set_add_all(\"__acc\", \"a1\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + new ExpressionLambdaAggregatorFactory( + "a2", + ImmutableSet.of("d1"), + "__acc", + "[]", + "[]", + "array_append(\"__acc\", \"d1\")", + "array_concat(\"__acc\", \"a2\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + new ExpressionLambdaAggregatorFactory( + "a3", + ImmutableSet.of("d1"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"d1\")", + "array_set_add_all(\"__acc\", \"a3\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + new ExpressionLambdaAggregatorFactory( + "a4", + ImmutableSet.of("f1"), + "__acc", + "[]", + "[]", + "array_append(\"__acc\", \"f1\")", + "array_concat(\"__acc\", \"a4\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + new ExpressionLambdaAggregatorFactory( + "a5", + ImmutableSet.of("f1"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"f1\")", + "array_set_add_all(\"__acc\", \"a5\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ) + ) + ) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + useDefault + ? new Object[]{ + "[7,325323,0,0,0,0]", + "[0,7,325323]", + "[1.0,1.7,0.0,0.0,0.0,0.0]", + "[1.0,0.0,1.7]", + "[1.0,0.10000000149011612,0.0,0.0,0.0,0.0]", + "[1.0,0.10000000149011612,0.0]" + } + : new Object[]{ + "[7,325323,0,null,null,null]", + "[0,null,7,325323]", + "[1.0,1.7,0.0,null,null,null]", + "[1.0,0.0,null,1.7]", + "[1.0,0.10000000149011612,0.0,null,null,null]", + "[1.0,0.10000000149011612,0.0,null]" + } + ) + ); + } + + @Test + public void testArrayAggToString() throws Exception + { + cannotVectorize(); + testQuery( + "SELECT ARRAY_TO_STRING(ARRAY_AGG(DISTINCT dim1), ',') FROM foo WHERE dim1 is not null", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(not(selector("dim1", null, null))) + .aggregators( + aggregators( + new ExpressionLambdaAggregatorFactory( + "a0", + ImmutableSet.of("dim1"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"dim1\")", + "array_set_add_all(\"__acc\", \"a0\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ) + ) + ) + .postAggregators(expressionPostAgg("p0", "array_to_string(\"a0\",',')")) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + useDefault ? new Object[]{"1,2,abc,def,10.1"} : new Object[]{",1,2,abc,def,10.1"} + ) + ); + } + + @Test + public void testArrayAggExpression() throws Exception + { + cannotVectorize(); + testQuery( + "SELECT ARRAY_TO_STRING(ARRAY_AGG(DISTINCT CONCAT(dim1, dim2)), ',') FROM foo", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .virtualColumns( + expressionVirtualColumn("v0", "concat(\"dim1\",\"dim2\")", ValueType.STRING) + ) + .aggregators( + aggregators( + new ExpressionLambdaAggregatorFactory( + "a0", + ImmutableSet.of("v0"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"v0\")", + "array_set_add_all(\"__acc\", \"a0\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ) + ) + ) + .postAggregators(expressionPostAgg("p0", "array_to_string(\"a0\",',')")) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + useDefault ? new Object[]{"1a,a,2,abc,10.1,defabc"} : new Object[]{"null,1a,a,2,defabc"} + ) + ); + } + + @Test + public void testArrayAggMaxBytes() throws Exception + { + cannotVectorize(); + testQuery( + "SELECT ARRAY_AGG(l1, 128), ARRAY_AGG(DISTINCT l1, 128) FROM numfoo", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .aggregators( + aggregators( + new ExpressionLambdaAggregatorFactory( + "a0", + ImmutableSet.of("l1"), + "__acc", + "[]", + "[]", + "array_append(\"__acc\", \"l1\")", + "array_concat(\"__acc\", \"a0\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(128), + TestExprMacroTable.INSTANCE + ), + new ExpressionLambdaAggregatorFactory( + "a1", + ImmutableSet.of("l1"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"l1\")", + "array_set_add_all(\"__acc\", \"a1\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(128), + TestExprMacroTable.INSTANCE + ) + ) + ) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + useDefault + ? new Object[]{"[7,325323,0,0,0,0]", "[0,7,325323]"} + : new Object[]{"[7,325323,0,null,null,null]", "[0,null,7,325323]"} + ) + ); + } + + @Test + public void testArrayAggAsArrayFromJoin() throws Exception + { + cannotVectorize(); + List expectedResults; + if (useDefault) { + expectedResults = ImmutableList.of( + new Object[]{"a", "[\"2\",\"10.1\"]", "2,10.1"}, + new Object[]{"a", "[\"2\",\"10.1\"]", "2,10.1"}, + new Object[]{"a", "[\"2\",\"10.1\"]", "2,10.1"}, + new Object[]{"b", "[\"1\",\"abc\",\"def\"]", "1,abc,def"}, + new Object[]{"b", "[\"1\",\"abc\",\"def\"]", "1,abc,def"}, + new Object[]{"b", "[\"1\",\"abc\",\"def\"]", "1,abc,def"} + ); + } else { + expectedResults = ImmutableList.of( + new Object[]{"a", "[\"\",\"2\",\"10.1\"]", ",2,10.1"}, + new Object[]{"a", "[\"\",\"2\",\"10.1\"]", ",2,10.1"}, + new Object[]{"a", "[\"\",\"2\",\"10.1\"]", ",2,10.1"}, + new Object[]{"b", "[\"1\",\"abc\",\"def\"]", "1,abc,def"}, + new Object[]{"b", "[\"1\",\"abc\",\"def\"]", "1,abc,def"}, + new Object[]{"b", "[\"1\",\"abc\",\"def\"]", "1,abc,def"} + ); + } + testQuery( + "SELECT numfoo.dim4, j.arr, ARRAY_TO_STRING(j.arr, ',') FROM numfoo INNER JOIN (SELECT dim4, ARRAY_AGG(DISTINCT dim1) as arr FROM numfoo WHERE dim1 is not null GROUP BY 1) as j ON numfoo.dim4 = j.dim4", + ImmutableList.of( + Druids.newScanQueryBuilder() + .dataSource( + join( + new TableDataSource(CalciteTests.DATASOURCE3), + new QueryDataSource( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setDimFilter(not(selector("dim1", null, null))) + .setDimensions(new DefaultDimensionSpec("dim4", "_d0")) + .setAggregatorSpecs( + aggregators( + new ExpressionLambdaAggregatorFactory( + "a0", + ImmutableSet.of("dim1"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"dim1\")", + "array_set_add_all(\"__acc\", \"a0\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ) + ) + ) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + "j0.", + "(\"dim4\" == \"j0._d0\")", + JoinType.INNER, + null + ) + ) + .virtualColumns( + expressionVirtualColumn("v0", "array_to_string(\"j0.a0\",',')", ValueType.STRING) + ) + .intervals(querySegmentSpec(Filtration.eternity())) + .columns("dim4", "j0.a0", "v0") + .context(QUERY_CONTEXT_DEFAULT) + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .legacy(false) + .build() + + ), + expectedResults + ); + } + + @Test + public void testArrayAggGroupByArrayAggFromSubquery() throws Exception + { + cannotVectorize(); + // yo, can't group on array types right now so expect failure + expectedException.expect(RuntimeException.class); + expectedException.expectMessage("Cannot create query type helper from invalid type [STRING_ARRAY]"); + testQuery( + "SELECT dim2, arr, COUNT(*) FROM (SELECT dim2, ARRAY_AGG(DISTINCT dim1) as arr FROM foo WHERE dim1 is not null GROUP BY 1 LIMIT 5) GROUP BY 1,2", + ImmutableList.of(), + ImmutableList.of() + ); + } + + @Test + public void testArrayAggArrayContainsSubquery() throws Exception + { + cannotVectorize(); + List expectedResults; + if (useDefault) { + expectedResults = ImmutableList.of( + new Object[]{"10.1", ""}, + new Object[]{"2", ""}, + new Object[]{"1", "a"}, + new Object[]{"def", "abc"}, + new Object[]{"abc", ""} + ); + } else { + expectedResults = ImmutableList.of( + new Object[]{"", "a"}, + new Object[]{"10.1", null}, + new Object[]{"2", ""}, + new Object[]{"1", "a"}, + new Object[]{"def", "abc"}, + new Object[]{"abc", null} + ); + } + testQuery( + "SELECT dim1,dim2 FROM foo WHERE ARRAY_CONTAINS((SELECT ARRAY_AGG(DISTINCT dim1) FROM foo WHERE dim1 is not null), dim1)", + ImmutableList.of( + Druids.newScanQueryBuilder() + .dataSource( + join( + new TableDataSource(CalciteTests.DATASOURCE1), + new QueryDataSource( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(not(selector("dim1", null, null))) + .aggregators( + aggregators( + new ExpressionLambdaAggregatorFactory( + "a0", + ImmutableSet.of("dim1"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"dim1\")", + "array_set_add_all(\"__acc\", \"a0\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ) + ) + ) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + "j0.", + "1", + JoinType.LEFT, + null + ) + ) + .intervals(querySegmentSpec(Filtration.eternity())) + .filters( + new ExpressionDimFilter( + "array_contains(\"j0.a0\",\"dim1\")", + TestExprMacroTable.INSTANCE + ) + ) + .columns("dim1", "dim2") + .context(QUERY_CONTEXT_DEFAULT) + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .legacy(false) + .build() + + ), + expectedResults + ); + } + + @Test + public void testArrayAggGroupByArrayContainsSubquery() throws Exception + { + cannotVectorize(); + List expectedResults; + if (useDefault) { + expectedResults = ImmutableList.of( + new Object[]{"", 3L}, + new Object[]{"a", 1L}, + new Object[]{"abc", 1L} + ); + } else { + expectedResults = ImmutableList.of( + new Object[]{null, 2L}, + new Object[]{"", 1L}, + new Object[]{"a", 2L}, + new Object[]{"abc", 1L} + ); + } + testQuery( + "SELECT dim2, COUNT(*) FROM foo WHERE ARRAY_CONTAINS((SELECT ARRAY_AGG(DISTINCT dim1) FROM foo WHERE dim1 is not null), dim1) GROUP BY 1", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource( + join( + new TableDataSource(CalciteTests.DATASOURCE1), + new QueryDataSource( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(not(selector("dim1", null, null))) + .aggregators( + aggregators( + new ExpressionLambdaAggregatorFactory( + "a0", + ImmutableSet.of("dim1"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"dim1\")", + "array_set_add_all(\"__acc\", \"a0\")", + null, + "if(array_length(o) == 0, null, o)", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ) + ) + ) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + "j0.", + "1", + JoinType.LEFT, + null + ) + ) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setDimFilter( + new ExpressionDimFilter( + "array_contains(\"j0.a0\",\"dim1\")", + TestExprMacroTable.INSTANCE + ) + ) + .setDimensions(dimensions(new DefaultDimensionSpec("dim2", "d0"))) + .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))) + .setGranularity(Granularities.ALL) + .setLimitSpec(NoopLimitSpec.instance()) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + + ), + expectedResults + ); + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java new file mode 100644 index 00000000000..62254683170 --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java @@ -0,0 +1,936 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite; + +import com.google.common.collect.ImmutableList; +import junitparams.JUnitParamsRunner; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.dimension.DefaultDimensionSpec; +import org.apache.druid.query.filter.AndDimFilter; +import org.apache.druid.query.filter.InDimFilter; +import org.apache.druid.query.filter.SelectorDimFilter; +import org.apache.druid.query.groupby.GroupByQuery; +import org.apache.druid.query.groupby.orderby.DefaultLimitSpec; +import org.apache.druid.query.groupby.orderby.OrderByColumnSpec; +import org.apache.druid.query.ordering.StringComparators; +import org.apache.druid.query.scan.ScanQuery; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.sql.calcite.filtration.Filtration; +import org.apache.druid.sql.calcite.util.CalciteTests; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.util.List; + +@RunWith(JUnitParamsRunner.class) +public class CalciteMultiValueStringQueryTest extends BaseCalciteQueryTest +{ + // various queries on multi-valued string dimensions using them like strings + @Test + public void testMultiValueStringWorksLikeStringGroupBy() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + List expected; + if (NullHandling.replaceWithDefault()) { + expected = ImmutableList.of( + new Object[]{"bfoo", 2L}, + new Object[]{"foo", 2L}, + new Object[]{"", 1L}, + new Object[]{"afoo", 1L}, + new Object[]{"cfoo", 1L}, + new Object[]{"dfoo", 1L} + ); + } else { + expected = ImmutableList.of( + new Object[]{null, 2L}, + new Object[]{"bfoo", 2L}, + new Object[]{"afoo", 1L}, + new Object[]{"cfoo", 1L}, + new Object[]{"dfoo", 1L}, + new Object[]{"foo", 1L} + ); + } + testQuery( + "SELECT concat(dim3, 'foo'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn("v0", "concat(\"dim3\",'foo')", ValueType.STRING)) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + expected + ); + } + + @Test + public void testMultiValueStringWorksLikeStringGroupByWithFilter() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + testQuery( + "SELECT concat(dim3, 'foo'), SUM(cnt) FROM druid.numfoo where concat(dim3, 'foo') = 'bfoo' GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn("v0", "concat(\"dim3\",'foo')", ValueType.STRING)) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) + ) + ) + .setDimFilter(selector("v0", "bfoo", null)) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{"bfoo", 2L}, + new Object[]{"afoo", 1L}, + new Object[]{"cfoo", 1L} + ) + ); + } + + @Test + public void testMultiValueStringWorksLikeStringScan() throws Exception + { + final String nullVal = NullHandling.replaceWithDefault() ? "[\"foo\"]" : "[null]"; + testQuery( + "SELECT concat(dim3, 'foo') FROM druid.numfoo", + ImmutableList.of( + new Druids.ScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .virtualColumns(expressionVirtualColumn("v0", "concat(\"dim3\",'foo')", ValueType.STRING)) + .columns(ImmutableList.of("v0")) + .context(QUERY_CONTEXT_DEFAULT) + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .legacy(false) + .build() + ), + ImmutableList.of( + new Object[]{"[\"afoo\",\"bfoo\"]"}, + new Object[]{"[\"bfoo\",\"cfoo\"]"}, + new Object[]{"[\"dfoo\"]"}, + new Object[]{"[\"foo\"]"}, + new Object[]{nullVal}, + new Object[]{nullVal} + ) + ); + } + + @Test + public void testMultiValueStringWorksLikeStringSelfConcatScan() throws Exception + { + final String nullVal = NullHandling.replaceWithDefault() ? "[\"-lol-\"]" : "[null]"; + testQuery( + "SELECT concat(dim3, '-lol-', dim3) FROM druid.numfoo", + ImmutableList.of( + new Druids.ScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .virtualColumns(expressionVirtualColumn("v0", "concat(\"dim3\",'-lol-',\"dim3\")", ValueType.STRING)) + .columns(ImmutableList.of("v0")) + .context(QUERY_CONTEXT_DEFAULT) + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .legacy(false) + .build() + ), + ImmutableList.of( + new Object[]{"[\"a-lol-a\",\"b-lol-b\"]"}, + new Object[]{"[\"b-lol-b\",\"c-lol-c\"]"}, + new Object[]{"[\"d-lol-d\"]"}, + new Object[]{"[\"-lol-\"]"}, + new Object[]{nullVal}, + new Object[]{nullVal} + ) + ); + } + + @Test + public void testMultiValueStringWorksLikeStringScanWithFilter() throws Exception + { + testQuery( + "SELECT concat(dim3, 'foo') FROM druid.numfoo where concat(dim3, 'foo') = 'bfoo'", + ImmutableList.of( + new Druids.ScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .virtualColumns(expressionVirtualColumn("v0", "concat(\"dim3\",'foo')", ValueType.STRING)) + .filters(selector("v0", "bfoo", null)) + .columns(ImmutableList.of("v0")) + .context(QUERY_CONTEXT_DEFAULT) + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .legacy(false) + .build() + ), + ImmutableList.of( + new Object[]{"[\"afoo\",\"bfoo\"]"}, + new Object[]{"[\"bfoo\",\"cfoo\"]"} + ) + ); + } + + // these are a copy of the ARRAY functions tests in CalciteArraysQueryTest + @Test + public void testMultiValueStringOverlapFilter() throws Exception + { + testQuery( + "SELECT dim3 FROM druid.numfoo WHERE MV_OVERLAP(dim3, ARRAY['a','b']) LIMIT 5", + ImmutableList.of( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .filters(new InDimFilter("dim3", ImmutableList.of("a", "b"), null)) + .columns("dim3") + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .limit(5) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{"[\"a\",\"b\"]"}, + new Object[]{"[\"b\",\"c\"]"} + ) + ); + } + + @Test + public void testMultiValueStringOverlapFilterNonLiteral() throws Exception + { + testQuery( + "SELECT dim3 FROM druid.numfoo WHERE MV_OVERLAP(dim3, ARRAY[dim2]) LIMIT 5", + ImmutableList.of( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .filters(expressionFilter("array_overlap(\"dim3\",array(\"dim2\"))")) + .columns("dim3") + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .limit(5) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{"[\"a\",\"b\"]"}, + new Object[]{useDefault ? "" : null} + ) + ); + } + + @Test + public void testMultiValueStringContainsFilter() throws Exception + { + testQuery( + "SELECT dim3 FROM druid.numfoo WHERE MV_CONTAINS(dim3, ARRAY['a','b']) LIMIT 5", + ImmutableList.of( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .filters( + new AndDimFilter( + new SelectorDimFilter("dim3", "a", null), + new SelectorDimFilter("dim3", "b", null) + ) + ) + .columns("dim3") + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .limit(5) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{"[\"a\",\"b\"]"} + ) + ); + } + + @Test + public void testMultiValueStringContainsArrayOfOneElement() throws Exception + { + testQuery( + "SELECT dim3 FROM druid.numfoo WHERE MV_CONTAINS(dim3, ARRAY['a']) LIMIT 5", + ImmutableList.of( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .filters(new SelectorDimFilter("dim3", "a", null)) + .columns("dim3") + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .limit(5) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{"[\"a\",\"b\"]"} + ) + ); + } + + @Test + public void testMultiValueStringContainsArrayOfNonLiteral() throws Exception + { + testQuery( + "SELECT dim3 FROM druid.numfoo WHERE MV_CONTAINS(dim3, ARRAY[dim2]) LIMIT 5", + ImmutableList.of( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .filters(expressionFilter("array_contains(\"dim3\",array(\"dim2\"))")) + .columns("dim3") + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .limit(5) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{"[\"a\",\"b\"]"}, + new Object[]{useDefault ? "" : null} + ) + ); + } + + @Test + public void testMultiValueStringSlice() throws Exception + { + testQuery( + "SELECT MV_SLICE(dim3, 1) FROM druid.numfoo", + ImmutableList.of( + new Druids.ScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .virtualColumns(expressionVirtualColumn("v0", "array_slice(\"dim3\",1)", ValueType.STRING)) + .columns(ImmutableList.of("v0")) + .context(QUERY_CONTEXT_DEFAULT) + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .legacy(false) + .build() + ), + ImmutableList.of( + new Object[]{"[\"b\"]"}, + new Object[]{"[\"c\"]"}, + new Object[]{"[]"}, + new Object[]{"[]"}, + new Object[]{"[]"}, + new Object[]{"[]"} + ) + ); + } + + @Test + public void testMultiValueStringLength() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + testQuery( + "SELECT dim1, MV_LENGTH(dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1, 2 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn("v0", "array_length(\"dim3\")", ValueType.LONG)) + .setDimensions( + dimensions( + new DefaultDimensionSpec("dim1", "_d0", ValueType.STRING), + new DefaultDimensionSpec("v0", "_d1", ValueType.LONG) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "_d1", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{"", 2, 1L}, + new Object[]{"10.1", 2, 1L}, + new Object[]{"1", 1, 1L}, + new Object[]{"2", 1, 1L}, + new Object[]{"abc", 1, 1L}, + new Object[]{"def", 1, 1L} + ) + ); + } + + @Test + public void testMultiValueStringAppend() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + ImmutableList results; + if (useDefault) { + results = ImmutableList.of( + new Object[]{"foo", 6L}, + new Object[]{"", 3L}, + new Object[]{"b", 2L}, + new Object[]{"a", 1L}, + new Object[]{"c", 1L}, + new Object[]{"d", 1L} + ); + } else { + results = ImmutableList.of( + new Object[]{"foo", 6L}, + new Object[]{null, 2L}, + new Object[]{"b", 2L}, + new Object[]{"", 1L}, + new Object[]{"a", 1L}, + new Object[]{"c", 1L}, + new Object[]{"d", 1L} + ); + } + testQuery( + "SELECT MV_APPEND(dim3, 'foo'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn( + "v0", + "array_append(\"dim3\",'foo')", + ValueType.STRING + )) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + results + ); + } + + @Test + public void testMultiValueStringPrepend() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + ImmutableList results; + if (useDefault) { + results = ImmutableList.of( + new Object[]{"foo", 6L}, + new Object[]{"", 3L}, + new Object[]{"b", 2L}, + new Object[]{"a", 1L}, + new Object[]{"c", 1L}, + new Object[]{"d", 1L} + ); + } else { + results = ImmutableList.of( + new Object[]{"foo", 6L}, + new Object[]{null, 2L}, + new Object[]{"b", 2L}, + new Object[]{"", 1L}, + new Object[]{"a", 1L}, + new Object[]{"c", 1L}, + new Object[]{"d", 1L} + ); + } + testQuery( + "SELECT MV_PREPEND('foo', dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn( + "v0", + "array_prepend('foo',\"dim3\")", + ValueType.STRING + )) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + results + ); + } + + @Test + public void testMultiValueStringPrependAppend() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + ImmutableList results; + if (useDefault) { + results = ImmutableList.of( + new Object[]{"foo,null", "null,foo", 3L}, + new Object[]{"foo,a,b", "a,b,foo", 1L}, + new Object[]{"foo,b,c", "b,c,foo", 1L}, + new Object[]{"foo,d", "d,foo", 1L} + ); + } else { + results = ImmutableList.of( + new Object[]{"foo,null", "null,foo", 2L}, + new Object[]{"foo,", ",foo", 1L}, + new Object[]{"foo,a,b", "a,b,foo", 1L}, + new Object[]{"foo,b,c", "b,c,foo", 1L}, + new Object[]{"foo,d", "d,foo", 1L} + ); + } + testQuery( + "SELECT MV_TO_STRING(MV_PREPEND('foo', dim3), ','), MV_TO_STRING(MV_APPEND(dim3, 'foo'), ','), SUM(cnt) FROM druid.numfoo GROUP BY 1,2 ORDER BY 3 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns( + expressionVirtualColumn( + "v0", + "array_to_string(array_prepend('foo',\"dim3\"),',')", + ValueType.STRING + ), + expressionVirtualColumn( + "v1", + "array_to_string(array_append(\"dim3\",'foo'),',')", + ValueType.STRING + ) + ) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING), + new DefaultDimensionSpec("v1", "_d1", ValueType.STRING) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + results + ); + } + + @Test + public void testMultiValueStringConcat() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + ImmutableList results; + if (useDefault) { + results = ImmutableList.of( + new Object[]{"", 6L}, + new Object[]{"b", 4L}, + new Object[]{"a", 2L}, + new Object[]{"c", 2L}, + new Object[]{"d", 2L} + ); + } else { + results = ImmutableList.of( + new Object[]{null, 4L}, + new Object[]{"b", 4L}, + new Object[]{"", 2L}, + new Object[]{"a", 2L}, + new Object[]{"c", 2L}, + new Object[]{"d", 2L} + ); + } + testQuery( + "SELECT MV_CONCAT(dim3, dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn( + "v0", + "array_concat(\"dim3\",\"dim3\")", + ValueType.STRING + )) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + results + ); + } + + @Test + public void testMultiValueStringOffset() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + testQuery( + "SELECT MV_OFFSET(dim3, 1), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn("v0", "array_offset(\"dim3\",1)", ValueType.STRING)) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{NullHandling.defaultStringValue(), 4L}, + new Object[]{"b", 1L}, + new Object[]{"c", 1L} + ) + ); + } + + @Test + public void testMultiValueStringOrdinal() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + testQuery( + "SELECT MV_ORDINAL(dim3, 2), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn("v0", "array_ordinal(\"dim3\",2)", ValueType.STRING)) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{NullHandling.defaultStringValue(), 4L}, + new Object[]{"b", 1L}, + new Object[]{"c", 1L} + ) + ); + } + + @Test + public void testMultiValueStringOffsetOf() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + testQuery( + "SELECT MV_OFFSET_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn( + "v0", + "array_offset_of(\"dim3\",'b')", + ValueType.LONG + )) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.LONG) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{useDefault ? -1 : null, 4L}, + new Object[]{0, 1L}, + new Object[]{1, 1L} + ) + ); + } + + @Test + public void testMultiValueStringOrdinalOf() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + testQuery( + "SELECT MV_ORDINAL_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn( + "v0", + "array_ordinal_of(\"dim3\",'b')", + ValueType.LONG + )) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.LONG) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{useDefault ? -1 : null, 4L}, + new Object[]{1, 1L}, + new Object[]{2, 1L} + ) + ); + } + + @Test + public void testMultiValueStringToString() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + ImmutableList results; + if (useDefault) { + results = ImmutableList.of( + new Object[]{"", 3L}, + new Object[]{"a,b", 1L}, + new Object[]{"b,c", 1L}, + new Object[]{"d", 1L} + ); + } else { + results = ImmutableList.of( + new Object[]{null, 2L}, + new Object[]{"", 1L}, + new Object[]{"a,b", 1L}, + new Object[]{"b,c", 1L}, + new Object[]{"d", 1L} + ); + } + testQuery( + "SELECT MV_TO_STRING(dim3, ','), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn( + "v0", + "array_to_string(\"dim3\",',')", + ValueType.STRING + )) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + results + ); + } + + @Test + public void testMultiValueStringToStringToMultiValueString() throws Exception + { + // Cannot vectorize due to usage of expressions. + cannotVectorize(); + + ImmutableList results; + if (useDefault) { + results = ImmutableList.of( + new Object[]{"d", 7L}, + new Object[]{"", 3L}, + new Object[]{"b", 2L}, + new Object[]{"a", 1L}, + new Object[]{"c", 1L} + ); + } else { + results = ImmutableList.of( + new Object[]{"d", 5L}, + new Object[]{null, 2L}, + new Object[]{"b", 2L}, + new Object[]{"", 1L}, + new Object[]{"a", 1L}, + new Object[]{"c", 1L} + ); + } + testQuery( + "SELECT STRING_TO_MV(CONCAT(MV_TO_STRING(dim3, ','), ',d'), ','), SUM(cnt) FROM druid.numfoo WHERE MV_LENGTH(dim3) > 0 GROUP BY 1 ORDER BY 2 DESC", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE3) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns( + expressionVirtualColumn("v0", "array_length(\"dim3\")", ValueType.LONG), + expressionVirtualColumn( + "v1", + "string_to_array(concat(array_to_string(\"dim3\",','),',d'),',')", + ValueType.STRING + ) + ) + .setDimFilter(bound("v0", "0", null, true, false, null, StringComparators.NUMERIC)) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v1", "_d0", ValueType.STRING) + ) + ) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setLimitSpec(new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec( + "a0", + OrderByColumnSpec.Direction.DESCENDING, + StringComparators.NUMERIC + )), + Integer.MAX_VALUE + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + results + ); + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteParameterQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteParameterQueryTest.java index e10b6fe7ea4..c23cd7da8eb 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteParameterQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteParameterQueryTest.java @@ -20,6 +20,7 @@ package org.apache.druid.sql.calcite; import com.google.common.collect.ImmutableList; +import junitparams.JUnitParamsRunner; import org.apache.calcite.avatica.SqlType; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.DateTimes; @@ -41,6 +42,7 @@ import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.http.SqlParameter; import org.junit.Test; +import org.junit.runner.RunWith; /** * This class has copied a subset of the tests in {@link CalciteQueryTest} and replaced various parts of queries with @@ -48,10 +50,9 @@ import org.junit.Test; * were merely chosen to produce a selection of parameter types and positions within query expressions and have been * renamed to reflect this */ +@RunWith(JUnitParamsRunner.class) public class CalciteParameterQueryTest extends BaseCalciteQueryTest { - private final boolean useDefault = NullHandling.replaceWithDefault(); - @Test public void testSelectConstantParamGetsConstant() throws Exception { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 976d61a0e20..9a3ad28c763 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -87,10 +87,8 @@ import org.apache.druid.query.dimension.ExtractionDimensionSpec; import org.apache.druid.query.expression.TestExprMacroTable; import org.apache.druid.query.extraction.RegexDimExtractionFn; import org.apache.druid.query.extraction.SubstringDimExtractionFn; -import org.apache.druid.query.filter.AndDimFilter; import org.apache.druid.query.filter.BoundDimFilter; import org.apache.druid.query.filter.DimFilter; -import org.apache.druid.query.filter.ExpressionDimFilter; import org.apache.druid.query.filter.InDimFilter; import org.apache.druid.query.filter.LikeDimFilter; import org.apache.druid.query.filter.NotDimFilter; @@ -101,7 +99,6 @@ import org.apache.druid.query.groupby.GroupByQuery.Builder; import org.apache.druid.query.groupby.GroupByQueryConfig; import org.apache.druid.query.groupby.ResultRow; import org.apache.druid.query.groupby.orderby.DefaultLimitSpec; -import org.apache.druid.query.groupby.orderby.NoopLimitSpec; import org.apache.druid.query.groupby.orderby.OrderByColumnSpec; import org.apache.druid.query.groupby.orderby.OrderByColumnSpec.Direction; import org.apache.druid.query.lookup.RegisteredLookupExtractionFn; @@ -151,8 +148,6 @@ import java.util.stream.Collectors; @RunWith(JUnitParamsRunner.class) public class CalciteQueryTest extends BaseCalciteQueryTest { - private final boolean useDefault = NullHandling.replaceWithDefault(); - @Test public void testSelectConstantExpression() throws Exception { @@ -16084,987 +16079,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ); } - @Test - public void testMultiValueStringWorksLikeStringGroupBy() throws Exception - { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - - List expected; - if (NullHandling.replaceWithDefault()) { - expected = ImmutableList.of( - new Object[]{"bfoo", 2L}, - new Object[]{"foo", 2L}, - new Object[]{"", 1L}, - new Object[]{"afoo", 1L}, - new Object[]{"cfoo", 1L}, - new Object[]{"dfoo", 1L} - ); - } else { - expected = ImmutableList.of( - new Object[]{null, 2L}, - new Object[]{"bfoo", 2L}, - new Object[]{"afoo", 1L}, - new Object[]{"cfoo", 1L}, - new Object[]{"dfoo", 1L}, - new Object[]{"foo", 1L} - ); - } - testQuery( - "SELECT concat(dim3, 'foo'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource(CalciteTests.DATASOURCE3) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setVirtualColumns(expressionVirtualColumn("v0", "concat(\"dim3\",'foo')", ValueType.STRING)) - .setDimensions( - dimensions( - new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) - ) - ) - .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) - .setLimitSpec(new DefaultLimitSpec( - ImmutableList.of(new OrderByColumnSpec( - "a0", - Direction.DESCENDING, - StringComparators.NUMERIC - )), - Integer.MAX_VALUE - )) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - expected - ); - } - - @Test - public void testMultiValueStringWorksLikeStringGroupByWithFilter() throws Exception - { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - - testQuery( - "SELECT concat(dim3, 'foo'), SUM(cnt) FROM druid.numfoo where concat(dim3, 'foo') = 'bfoo' GROUP BY 1 ORDER BY 2 DESC", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource(CalciteTests.DATASOURCE3) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setVirtualColumns(expressionVirtualColumn("v0", "concat(\"dim3\",'foo')", ValueType.STRING)) - .setDimensions( - dimensions( - new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) - ) - ) - .setDimFilter(selector("v0", "bfoo", null)) - .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) - .setLimitSpec(new DefaultLimitSpec( - ImmutableList.of(new OrderByColumnSpec( - "a0", - Direction.DESCENDING, - StringComparators.NUMERIC - )), - Integer.MAX_VALUE - )) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - new Object[]{"bfoo", 2L}, - new Object[]{"afoo", 1L}, - new Object[]{"cfoo", 1L} - ) - ); - } - - @Test - public void testMultiValueStringWorksLikeStringScan() throws Exception - { - final String nullVal = NullHandling.replaceWithDefault() ? "[\"foo\"]" : "[null]"; - testQuery( - "SELECT concat(dim3, 'foo') FROM druid.numfoo", - ImmutableList.of( - new Druids.ScanQueryBuilder() - .dataSource(CalciteTests.DATASOURCE3) - .intervals(querySegmentSpec(Filtration.eternity())) - .virtualColumns(expressionVirtualColumn("v0", "concat(\"dim3\",'foo')", ValueType.STRING)) - .columns(ImmutableList.of("v0")) - .context(QUERY_CONTEXT_DEFAULT) - .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) - .legacy(false) - .build() - ), - ImmutableList.of( - new Object[]{"[\"afoo\",\"bfoo\"]"}, - new Object[]{"[\"bfoo\",\"cfoo\"]"}, - new Object[]{"[\"dfoo\"]"}, - new Object[]{"[\"foo\"]"}, - new Object[]{nullVal}, - new Object[]{nullVal} - ) - ); - } - - @Test - public void testMultiValueStringWorksLikeStringSelfConcatScan() throws Exception - { - final String nullVal = NullHandling.replaceWithDefault() ? "[\"-lol-\"]" : "[null]"; - testQuery( - "SELECT concat(dim3, '-lol-', dim3) FROM druid.numfoo", - ImmutableList.of( - new Druids.ScanQueryBuilder() - .dataSource(CalciteTests.DATASOURCE3) - .intervals(querySegmentSpec(Filtration.eternity())) - .virtualColumns(expressionVirtualColumn("v0", "concat(\"dim3\",'-lol-',\"dim3\")", ValueType.STRING)) - .columns(ImmutableList.of("v0")) - .context(QUERY_CONTEXT_DEFAULT) - .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) - .legacy(false) - .build() - ), - ImmutableList.of( - new Object[]{"[\"a-lol-a\",\"b-lol-b\"]"}, - new Object[]{"[\"b-lol-b\",\"c-lol-c\"]"}, - new Object[]{"[\"d-lol-d\"]"}, - new Object[]{"[\"-lol-\"]"}, - new Object[]{nullVal}, - new Object[]{nullVal} - ) - ); - } - - @Test - public void testMultiValueStringWorksLikeStringScanWithFilter() throws Exception - { - testQuery( - "SELECT concat(dim3, 'foo') FROM druid.numfoo where concat(dim3, 'foo') = 'bfoo'", - ImmutableList.of( - new Druids.ScanQueryBuilder() - .dataSource(CalciteTests.DATASOURCE3) - .intervals(querySegmentSpec(Filtration.eternity())) - .virtualColumns(expressionVirtualColumn("v0", "concat(\"dim3\",'foo')", ValueType.STRING)) - .filters(selector("v0", "bfoo", null)) - .columns(ImmutableList.of("v0")) - .context(QUERY_CONTEXT_DEFAULT) - .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) - .legacy(false) - .build() - ), - ImmutableList.of( - new Object[]{"[\"afoo\",\"bfoo\"]"}, - new Object[]{"[\"bfoo\",\"cfoo\"]"} - ) - ); - } - - @Test - public void testSelectConstantArrayExpressionFromTable() throws Exception - { - testQuery( - "SELECT ARRAY[1,2] as arr, dim1 FROM foo LIMIT 1", - ImmutableList.of( - newScanQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(querySegmentSpec(Filtration.eternity())) - .virtualColumns(expressionVirtualColumn("v0", "array(1,2)", ValueType.STRING)) - .columns("dim1", "v0") - .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) - .limit(1) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - new Object[]{"[1,2]", ""} - ) - ); - } - - @Test - public void testGroupByArrayFromCase() throws Exception - { - cannotVectorize(); - testQuery( - "SELECT CASE WHEN dim4 = 'a' THEN ARRAY['foo','bar','baz'] END as mv_value, count(1) from numfoo GROUP BY 1", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource(CalciteTests.DATASOURCE3) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setVirtualColumns(expressionVirtualColumn("v0", "case_searched((\"dim4\" == 'a'),array('foo','bar','baz'),null)", ValueType.STRING)) - .setDimensions(new DefaultDimensionSpec("v0", "_d0")) - .setGranularity(Granularities.ALL) - .setAggregatorSpecs(new CountAggregatorFactory("a0")) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - new Object[]{null, 3L}, - new Object[]{"bar", 3L}, - new Object[]{"baz", 3L}, - new Object[]{"foo", 3L} - ) - ); - } - - @Test - public void testSelectNonConstantArrayExpressionFromTable() throws Exception - { - testQuery( - "SELECT ARRAY[CONCAT(dim1, 'word'),'up'] as arr, dim1 FROM foo LIMIT 5", - ImmutableList.of( - newScanQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(querySegmentSpec(Filtration.eternity())) - .virtualColumns(expressionVirtualColumn("v0", "array(concat(\"dim1\",'word'),'up')", ValueType.STRING)) - .columns("dim1", "v0") - .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) - .limit(5) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - new Object[]{"[\"word\",\"up\"]", ""}, - new Object[]{"[\"10.1word\",\"up\"]", "10.1"}, - new Object[]{"[\"2word\",\"up\"]", "2"}, - new Object[]{"[\"1word\",\"up\"]", "1"}, - new Object[]{"[\"defword\",\"up\"]", "def"} - ) - ); - } - - @Test - public void testSelectNonConstantArrayExpressionFromTableFailForMultival() throws Exception - { - // without expression output type inference to prevent this, the automatic translation will try to turn this into - // - // `map((dim3) -> array(concat(dim3,'word'),'up'), dim3)` - // - // This error message will get better in the future. The error without translation would be: - // - // org.apache.druid.java.util.common.RE: Unhandled array constructor element type [STRING_ARRAY] - - expectedException.expect(RuntimeException.class); - expectedException.expectMessage("Unhandled map function output type [STRING_ARRAY]"); - testQuery( - "SELECT ARRAY[CONCAT(dim3, 'word'),'up'] as arr, dim1 FROM foo LIMIT 5", - ImmutableList.of(), - ImmutableList.of() - ); - } - - @Test - public void testMultiValueStringOverlapFilter() throws Exception - { - testQuery( - "SELECT dim3 FROM druid.numfoo WHERE MV_OVERLAP(dim3, ARRAY['a','b']) LIMIT 5", - ImmutableList.of( - newScanQueryBuilder() - .dataSource(CalciteTests.DATASOURCE3) - .intervals(querySegmentSpec(Filtration.eternity())) - .filters(new InDimFilter("dim3", ImmutableList.of("a", "b"), null)) - .columns("dim3") - .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) - .limit(5) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - new Object[]{"[\"a\",\"b\"]"}, - new Object[]{"[\"b\",\"c\"]"} - ) - ); - } - - @Test - public void testMultiValueStringOverlapFilterNonLiteral() throws Exception - { - testQuery( - "SELECT dim3 FROM druid.numfoo WHERE MV_OVERLAP(dim3, ARRAY[dim2]) LIMIT 5", - ImmutableList.of( - newScanQueryBuilder() - .dataSource(CalciteTests.DATASOURCE3) - .intervals(querySegmentSpec(Filtration.eternity())) - .filters(expressionFilter("array_overlap(\"dim3\",array(\"dim2\"))")) - .columns("dim3") - .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) - .limit(5) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - new Object[]{"[\"a\",\"b\"]"}, - new Object[]{useDefault ? "" : null} - ) - ); - } - - @Test - public void testMultiValueStringContainsFilter() throws Exception - { - testQuery( - "SELECT dim3 FROM druid.numfoo WHERE MV_CONTAINS(dim3, ARRAY['a','b']) LIMIT 5", - ImmutableList.of( - newScanQueryBuilder() - .dataSource(CalciteTests.DATASOURCE3) - .intervals(querySegmentSpec(Filtration.eternity())) - .filters( - new AndDimFilter( - new SelectorDimFilter("dim3", "a", null), - new SelectorDimFilter("dim3", "b", null) - ) - ) - .columns("dim3") - .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) - .limit(5) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - new Object[]{"[\"a\",\"b\"]"} - ) - ); - } - - @Test - public void testMultiValueStringContainsArrayOfOneElement() throws Exception - { - testQuery( - "SELECT dim3 FROM druid.numfoo WHERE MV_CONTAINS(dim3, ARRAY['a']) LIMIT 5", - ImmutableList.of( - newScanQueryBuilder() - .dataSource(CalciteTests.DATASOURCE3) - .intervals(querySegmentSpec(Filtration.eternity())) - .filters(new SelectorDimFilter("dim3", "a", null)) - .columns("dim3") - .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) - .limit(5) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - new Object[]{"[\"a\",\"b\"]"} - ) - ); - } - - @Test - public void testMultiValueStringContainsArrayOfNonLiteral() throws Exception - { - testQuery( - "SELECT dim3 FROM druid.numfoo WHERE MV_CONTAINS(dim3, ARRAY[dim2]) LIMIT 5", - ImmutableList.of( - newScanQueryBuilder() - .dataSource(CalciteTests.DATASOURCE3) - .intervals(querySegmentSpec(Filtration.eternity())) - .filters(expressionFilter("array_contains(\"dim3\",array(\"dim2\"))")) - .columns("dim3") - .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) - .limit(5) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - new Object[]{"[\"a\",\"b\"]"}, - new Object[]{useDefault ? "" : null} - ) - ); - } - - @Test - public void testMultiValueStringSlice() throws Exception - { - testQuery( - "SELECT MV_SLICE(dim3, 1) FROM druid.numfoo", - ImmutableList.of( - new Druids.ScanQueryBuilder() - .dataSource(CalciteTests.DATASOURCE3) - .intervals(querySegmentSpec(Filtration.eternity())) - .virtualColumns(expressionVirtualColumn("v0", "array_slice(\"dim3\",1)", ValueType.STRING)) - .columns(ImmutableList.of("v0")) - .context(QUERY_CONTEXT_DEFAULT) - .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) - .legacy(false) - .build() - ), - ImmutableList.of( - new Object[]{"[\"b\"]"}, - new Object[]{"[\"c\"]"}, - new Object[]{"[]"}, - new Object[]{"[]"}, - new Object[]{"[]"}, - new Object[]{"[]"} - ) - ); - } - - @Test - public void testMultiValueStringLength() throws Exception - { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - - testQuery( - "SELECT dim1, MV_LENGTH(dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1, 2 ORDER BY 2 DESC", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource(CalciteTests.DATASOURCE3) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setVirtualColumns(expressionVirtualColumn("v0", "array_length(\"dim3\")", ValueType.LONG)) - .setDimensions( - dimensions( - new DefaultDimensionSpec("dim1", "_d0", ValueType.STRING), - new DefaultDimensionSpec("v0", "_d1", ValueType.LONG) - ) - ) - .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) - .setLimitSpec(new DefaultLimitSpec( - ImmutableList.of(new OrderByColumnSpec( - "_d1", - Direction.DESCENDING, - StringComparators.NUMERIC - )), - Integer.MAX_VALUE - )) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - new Object[]{"", 2, 1L}, - new Object[]{"10.1", 2, 1L}, - new Object[]{"1", 1, 1L}, - new Object[]{"2", 1, 1L}, - new Object[]{"abc", 1, 1L}, - new Object[]{"def", 1, 1L} - ) - ); - } - - @Test - public void testMultiValueStringAppend() throws Exception - { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - - ImmutableList results; - if (NullHandling.replaceWithDefault()) { - results = ImmutableList.of( - new Object[]{"foo", 6L}, - new Object[]{"", 3L}, - new Object[]{"b", 2L}, - new Object[]{"a", 1L}, - new Object[]{"c", 1L}, - new Object[]{"d", 1L} - ); - } else { - results = ImmutableList.of( - new Object[]{"foo", 6L}, - new Object[]{null, 2L}, - new Object[]{"b", 2L}, - new Object[]{"", 1L}, - new Object[]{"a", 1L}, - new Object[]{"c", 1L}, - new Object[]{"d", 1L} - ); - } - testQuery( - "SELECT MV_APPEND(dim3, 'foo'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource(CalciteTests.DATASOURCE3) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setVirtualColumns(expressionVirtualColumn( - "v0", - "array_append(\"dim3\",'foo')", - ValueType.STRING - )) - .setDimensions( - dimensions( - new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) - ) - ) - .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) - .setLimitSpec(new DefaultLimitSpec( - ImmutableList.of(new OrderByColumnSpec( - "a0", - Direction.DESCENDING, - StringComparators.NUMERIC - )), - Integer.MAX_VALUE - )) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - results - ); - } - - @Test - public void testMultiValueStringPrepend() throws Exception - { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - - ImmutableList results; - if (NullHandling.replaceWithDefault()) { - results = ImmutableList.of( - new Object[]{"foo", 6L}, - new Object[]{"", 3L}, - new Object[]{"b", 2L}, - new Object[]{"a", 1L}, - new Object[]{"c", 1L}, - new Object[]{"d", 1L} - ); - } else { - results = ImmutableList.of( - new Object[]{"foo", 6L}, - new Object[]{null, 2L}, - new Object[]{"b", 2L}, - new Object[]{"", 1L}, - new Object[]{"a", 1L}, - new Object[]{"c", 1L}, - new Object[]{"d", 1L} - ); - } - testQuery( - "SELECT MV_PREPEND('foo', dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource(CalciteTests.DATASOURCE3) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setVirtualColumns(expressionVirtualColumn( - "v0", - "array_prepend('foo',\"dim3\")", - ValueType.STRING - )) - .setDimensions( - dimensions( - new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) - ) - ) - .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) - .setLimitSpec(new DefaultLimitSpec( - ImmutableList.of(new OrderByColumnSpec( - "a0", - Direction.DESCENDING, - StringComparators.NUMERIC - )), - Integer.MAX_VALUE - )) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - results - ); - } - - @Test - public void testMultiValueStringPrependAppend() throws Exception - { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - - ImmutableList results; - if (NullHandling.replaceWithDefault()) { - results = ImmutableList.of( - new Object[]{"foo,null", "null,foo", 3L}, - new Object[]{"foo,a,b", "a,b,foo", 1L}, - new Object[]{"foo,b,c", "b,c,foo", 1L}, - new Object[]{"foo,d", "d,foo", 1L} - ); - } else { - results = ImmutableList.of( - new Object[]{"foo,null", "null,foo", 2L}, - new Object[]{"foo,", ",foo", 1L}, - new Object[]{"foo,a,b", "a,b,foo", 1L}, - new Object[]{"foo,b,c", "b,c,foo", 1L}, - new Object[]{"foo,d", "d,foo", 1L} - ); - } - testQuery( - "SELECT MV_TO_STRING(MV_PREPEND('foo', dim3), ','), MV_TO_STRING(MV_APPEND(dim3, 'foo'), ','), SUM(cnt) FROM druid.numfoo GROUP BY 1,2 ORDER BY 3 DESC", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource(CalciteTests.DATASOURCE3) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setVirtualColumns( - expressionVirtualColumn( - "v0", - "array_to_string(array_prepend('foo',\"dim3\"),',')", - ValueType.STRING - ), - expressionVirtualColumn( - "v1", - "array_to_string(array_append(\"dim3\",'foo'),',')", - ValueType.STRING - ) - ) - .setDimensions( - dimensions( - new DefaultDimensionSpec("v0", "_d0", ValueType.STRING), - new DefaultDimensionSpec("v1", "_d1", ValueType.STRING) - ) - ) - .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) - .setLimitSpec(new DefaultLimitSpec( - ImmutableList.of(new OrderByColumnSpec( - "a0", - Direction.DESCENDING, - StringComparators.NUMERIC - )), - Integer.MAX_VALUE - )) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - results - ); - } - - @Test - public void testMultiValueStringConcat() throws Exception - { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - - ImmutableList results; - if (NullHandling.replaceWithDefault()) { - results = ImmutableList.of( - new Object[]{"", 6L}, - new Object[]{"b", 4L}, - new Object[]{"a", 2L}, - new Object[]{"c", 2L}, - new Object[]{"d", 2L} - ); - } else { - results = ImmutableList.of( - new Object[]{null, 4L}, - new Object[]{"b", 4L}, - new Object[]{"", 2L}, - new Object[]{"a", 2L}, - new Object[]{"c", 2L}, - new Object[]{"d", 2L} - ); - } - testQuery( - "SELECT MV_CONCAT(dim3, dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource(CalciteTests.DATASOURCE3) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setVirtualColumns(expressionVirtualColumn( - "v0", - "array_concat(\"dim3\",\"dim3\")", - ValueType.STRING - )) - .setDimensions( - dimensions( - new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) - ) - ) - .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) - .setLimitSpec(new DefaultLimitSpec( - ImmutableList.of(new OrderByColumnSpec( - "a0", - Direction.DESCENDING, - StringComparators.NUMERIC - )), - Integer.MAX_VALUE - )) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - results - ); - } - - @Test - public void testMultiValueStringOffset() throws Exception - { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - - testQuery( - "SELECT MV_OFFSET(dim3, 1), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource(CalciteTests.DATASOURCE3) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setVirtualColumns(expressionVirtualColumn("v0", "array_offset(\"dim3\",1)", ValueType.STRING)) - .setDimensions( - dimensions( - new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) - ) - ) - .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) - .setLimitSpec(new DefaultLimitSpec( - ImmutableList.of(new OrderByColumnSpec( - "a0", - Direction.DESCENDING, - StringComparators.NUMERIC - )), - Integer.MAX_VALUE - )) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - new Object[]{NullHandling.defaultStringValue(), 4L}, - new Object[]{"b", 1L}, - new Object[]{"c", 1L} - ) - ); - } - - @Test - public void testMultiValueStringOrdinal() throws Exception - { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - - testQuery( - "SELECT MV_ORDINAL(dim3, 2), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource(CalciteTests.DATASOURCE3) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setVirtualColumns(expressionVirtualColumn("v0", "array_ordinal(\"dim3\",2)", ValueType.STRING)) - .setDimensions( - dimensions( - new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) - ) - ) - .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) - .setLimitSpec(new DefaultLimitSpec( - ImmutableList.of(new OrderByColumnSpec( - "a0", - Direction.DESCENDING, - StringComparators.NUMERIC - )), - Integer.MAX_VALUE - )) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - new Object[]{NullHandling.defaultStringValue(), 4L}, - new Object[]{"b", 1L}, - new Object[]{"c", 1L} - ) - ); - } - - @Test - public void testMultiValueStringOffsetOf() throws Exception - { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - - testQuery( - "SELECT MV_OFFSET_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource(CalciteTests.DATASOURCE3) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setVirtualColumns(expressionVirtualColumn( - "v0", - "array_offset_of(\"dim3\",'b')", - ValueType.LONG - )) - .setDimensions( - dimensions( - new DefaultDimensionSpec("v0", "_d0", ValueType.LONG) - ) - ) - .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) - .setLimitSpec(new DefaultLimitSpec( - ImmutableList.of(new OrderByColumnSpec( - "a0", - Direction.DESCENDING, - StringComparators.NUMERIC - )), - Integer.MAX_VALUE - )) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - new Object[]{NullHandling.replaceWithDefault() ? -1 : null, 4L}, - new Object[]{0, 1L}, - new Object[]{1, 1L} - ) - ); - } - - @Test - public void testMultiValueStringOrdinalOf() throws Exception - { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - - testQuery( - "SELECT MV_ORDINAL_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource(CalciteTests.DATASOURCE3) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setVirtualColumns(expressionVirtualColumn( - "v0", - "array_ordinal_of(\"dim3\",'b')", - ValueType.LONG - )) - .setDimensions( - dimensions( - new DefaultDimensionSpec("v0", "_d0", ValueType.LONG) - ) - ) - .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) - .setLimitSpec(new DefaultLimitSpec( - ImmutableList.of(new OrderByColumnSpec( - "a0", - Direction.DESCENDING, - StringComparators.NUMERIC - )), - Integer.MAX_VALUE - )) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - new Object[]{NullHandling.replaceWithDefault() ? -1 : null, 4L}, - new Object[]{1, 1L}, - new Object[]{2, 1L} - ) - ); - } - - @Test - public void testMultiValueStringToString() throws Exception - { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - - ImmutableList results; - if (NullHandling.replaceWithDefault()) { - results = ImmutableList.of( - new Object[]{"", 3L}, - new Object[]{"a,b", 1L}, - new Object[]{"b,c", 1L}, - new Object[]{"d", 1L} - ); - } else { - results = ImmutableList.of( - new Object[]{null, 2L}, - new Object[]{"", 1L}, - new Object[]{"a,b", 1L}, - new Object[]{"b,c", 1L}, - new Object[]{"d", 1L} - ); - } - testQuery( - "SELECT MV_TO_STRING(dim3, ','), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource(CalciteTests.DATASOURCE3) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setVirtualColumns(expressionVirtualColumn( - "v0", - "array_to_string(\"dim3\",',')", - ValueType.STRING - )) - .setDimensions( - dimensions( - new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) - ) - ) - .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) - .setLimitSpec(new DefaultLimitSpec( - ImmutableList.of(new OrderByColumnSpec( - "a0", - Direction.DESCENDING, - StringComparators.NUMERIC - )), - Integer.MAX_VALUE - )) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - results - ); - } - - @Test - public void testMultiValueStringToStringToMultiValueString() throws Exception - { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - - ImmutableList results; - if (NullHandling.replaceWithDefault()) { - results = ImmutableList.of( - new Object[]{"d", 7L}, - new Object[]{"", 3L}, - new Object[]{"b", 2L}, - new Object[]{"a", 1L}, - new Object[]{"c", 1L} - ); - } else { - results = ImmutableList.of( - new Object[]{"d", 5L}, - new Object[]{null, 2L}, - new Object[]{"b", 2L}, - new Object[]{"", 1L}, - new Object[]{"a", 1L}, - new Object[]{"c", 1L} - ); - } - testQuery( - "SELECT STRING_TO_MV(CONCAT(MV_TO_STRING(dim3, ','), ',d'), ','), SUM(cnt) FROM druid.numfoo WHERE MV_LENGTH(dim3) > 0 GROUP BY 1 ORDER BY 2 DESC", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource(CalciteTests.DATASOURCE3) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setVirtualColumns( - expressionVirtualColumn("v0", "array_length(\"dim3\")", ValueType.LONG), - expressionVirtualColumn( - "v1", - "string_to_array(concat(array_to_string(\"dim3\",','),',d'),',')", - ValueType.STRING - ) - ) - .setDimFilter(bound("v0", "0", null, true, false, null, StringComparators.NUMERIC)) - .setDimensions( - dimensions( - new DefaultDimensionSpec("v1", "_d0", ValueType.STRING) - ) - ) - .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) - .setLimitSpec(new DefaultLimitSpec( - ImmutableList.of(new OrderByColumnSpec( - "a0", - Direction.DESCENDING, - StringComparators.NUMERIC - )), - Integer.MAX_VALUE - )) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - results - ); - } - @Test public void testLeftRightStringOperators() throws Exception { @@ -18309,627 +17323,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ); } - @Test - public void testArrayAgg() throws Exception - { - cannotVectorize(); - testQuery( - "SELECT ARRAY_AGG(dim1), ARRAY_AGG(DISTINCT dim1), ARRAY_AGG(DISTINCT dim1) FILTER(WHERE dim1 = 'shazbot') FROM foo WHERE dim1 is not null", - ImmutableList.of( - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(querySegmentSpec(Filtration.eternity())) - .granularity(Granularities.ALL) - .filters(not(selector("dim1", null, null))) - .aggregators( - aggregators( - new ExpressionLambdaAggregatorFactory( - "a0", - ImmutableSet.of("dim1"), - "__acc", - "[]", - "[]", - "array_append(\"__acc\", \"dim1\")", - "array_concat(\"__acc\", \"a0\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(1024), - TestExprMacroTable.INSTANCE - ), - new ExpressionLambdaAggregatorFactory( - "a1", - ImmutableSet.of("dim1"), - "__acc", - "[]", - "[]", - "array_set_add(\"__acc\", \"dim1\")", - "array_set_add_all(\"__acc\", \"a1\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(1024), - TestExprMacroTable.INSTANCE - ), - new FilteredAggregatorFactory( - new ExpressionLambdaAggregatorFactory( - "a2", - ImmutableSet.of("dim1"), - "__acc", - "[]", - "[]", - "array_set_add(\"__acc\", \"dim1\")", - "array_set_add_all(\"__acc\", \"a2\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(1024), - TestExprMacroTable.INSTANCE - ), - selector("dim1", "shazbot", null) - ) - ) - ) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - useDefault - ? new Object[]{"[\"10.1\",\"2\",\"1\",\"def\",\"abc\"]", "[\"1\",\"2\",\"abc\",\"def\",\"10.1\"]", null} - : new Object[]{"[\"\",\"10.1\",\"2\",\"1\",\"def\",\"abc\"]", "[\"\",\"1\",\"2\",\"abc\",\"def\",\"10.1\"]", null} - ) - ); - } - - @Test - public void testArrayAggMultiValue() throws Exception - { - cannotVectorize(); - testQuery( - "SELECT ARRAY_AGG(dim3), ARRAY_AGG(DISTINCT dim3) FROM foo", - ImmutableList.of( - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(querySegmentSpec(Filtration.eternity())) - .granularity(Granularities.ALL) - .aggregators( - aggregators( - new ExpressionLambdaAggregatorFactory( - "a0", - ImmutableSet.of("dim3"), - "__acc", - "[]", - "[]", - "array_append(\"__acc\", \"dim3\")", - "array_concat(\"__acc\", \"a0\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(1024), - TestExprMacroTable.INSTANCE - ), - new ExpressionLambdaAggregatorFactory( - "a1", - ImmutableSet.of("dim3"), - "__acc", - "[]", - "[]", - "array_set_add(\"__acc\", \"dim3\")", - "array_set_add_all(\"__acc\", \"a1\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(1024), - TestExprMacroTable.INSTANCE - ) - ) - ) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - useDefault - ? new Object[]{"[\"a\",\"b\",\"b\",\"c\",\"d\",null,null,null]", "[null,\"a\",\"b\",\"c\",\"d\"]"} - : new Object[]{"[\"a\",\"b\",\"b\",\"c\",\"d\",\"\",null,null]", "[\"\",null,\"a\",\"b\",\"c\",\"d\"]"} - ) - ); - } - - @Test - public void testArrayAggNumeric() throws Exception - { - cannotVectorize(); - testQuery( - "SELECT ARRAY_AGG(l1), ARRAY_AGG(DISTINCT l1), ARRAY_AGG(d1), ARRAY_AGG(DISTINCT d1), ARRAY_AGG(f1), ARRAY_AGG(DISTINCT f1) FROM numfoo", - ImmutableList.of( - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE3) - .intervals(querySegmentSpec(Filtration.eternity())) - .granularity(Granularities.ALL) - .aggregators( - aggregators( - new ExpressionLambdaAggregatorFactory( - "a0", - ImmutableSet.of("l1"), - "__acc", - "[]", - "[]", - "array_append(\"__acc\", \"l1\")", - "array_concat(\"__acc\", \"a0\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(1024), - TestExprMacroTable.INSTANCE - ), - new ExpressionLambdaAggregatorFactory( - "a1", - ImmutableSet.of("l1"), - "__acc", - "[]", - "[]", - "array_set_add(\"__acc\", \"l1\")", - "array_set_add_all(\"__acc\", \"a1\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(1024), - TestExprMacroTable.INSTANCE - ), - new ExpressionLambdaAggregatorFactory( - "a2", - ImmutableSet.of("d1"), - "__acc", - "[]", - "[]", - "array_append(\"__acc\", \"d1\")", - "array_concat(\"__acc\", \"a2\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(1024), - TestExprMacroTable.INSTANCE - ), - new ExpressionLambdaAggregatorFactory( - "a3", - ImmutableSet.of("d1"), - "__acc", - "[]", - "[]", - "array_set_add(\"__acc\", \"d1\")", - "array_set_add_all(\"__acc\", \"a3\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(1024), - TestExprMacroTable.INSTANCE - ), - new ExpressionLambdaAggregatorFactory( - "a4", - ImmutableSet.of("f1"), - "__acc", - "[]", - "[]", - "array_append(\"__acc\", \"f1\")", - "array_concat(\"__acc\", \"a4\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(1024), - TestExprMacroTable.INSTANCE - ), - new ExpressionLambdaAggregatorFactory( - "a5", - ImmutableSet.of("f1"), - "__acc", - "[]", - "[]", - "array_set_add(\"__acc\", \"f1\")", - "array_set_add_all(\"__acc\", \"a5\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(1024), - TestExprMacroTable.INSTANCE - ) - ) - ) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - useDefault - ? new Object[]{ - "[7,325323,0,0,0,0]", - "[0,7,325323]", - "[1.0,1.7,0.0,0.0,0.0,0.0]", - "[1.0,0.0,1.7]", - "[1.0,0.10000000149011612,0.0,0.0,0.0,0.0]", - "[1.0,0.10000000149011612,0.0]" - } - : new Object[]{ - "[7,325323,0,null,null,null]", - "[0,null,7,325323]", - "[1.0,1.7,0.0,null,null,null]", - "[1.0,0.0,null,1.7]", - "[1.0,0.10000000149011612,0.0,null,null,null]", - "[1.0,0.10000000149011612,0.0,null]" - } - ) - ); - } - - @Test - public void testArrayAggToString() throws Exception - { - cannotVectorize(); - testQuery( - "SELECT ARRAY_TO_STRING(ARRAY_AGG(DISTINCT dim1), ',') FROM foo WHERE dim1 is not null", - ImmutableList.of( - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(querySegmentSpec(Filtration.eternity())) - .granularity(Granularities.ALL) - .filters(not(selector("dim1", null, null))) - .aggregators( - aggregators( - new ExpressionLambdaAggregatorFactory( - "a0", - ImmutableSet.of("dim1"), - "__acc", - "[]", - "[]", - "array_set_add(\"__acc\", \"dim1\")", - "array_set_add_all(\"__acc\", \"a0\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(1024), - TestExprMacroTable.INSTANCE - ) - ) - ) - .postAggregators(expressionPostAgg("p0", "array_to_string(\"a0\",',')")) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - useDefault ? new Object[]{"1,2,abc,def,10.1"} : new Object[]{",1,2,abc,def,10.1"} - ) - ); - } - - @Test - public void testArrayAggExpression() throws Exception - { - cannotVectorize(); - testQuery( - "SELECT ARRAY_TO_STRING(ARRAY_AGG(DISTINCT CONCAT(dim1, dim2)), ',') FROM foo", - ImmutableList.of( - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(querySegmentSpec(Filtration.eternity())) - .granularity(Granularities.ALL) - .virtualColumns( - expressionVirtualColumn("v0", "concat(\"dim1\",\"dim2\")", ValueType.STRING) - ) - .aggregators( - aggregators( - new ExpressionLambdaAggregatorFactory( - "a0", - ImmutableSet.of("v0"), - "__acc", - "[]", - "[]", - "array_set_add(\"__acc\", \"v0\")", - "array_set_add_all(\"__acc\", \"a0\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(1024), - TestExprMacroTable.INSTANCE - ) - ) - ) - .postAggregators(expressionPostAgg("p0", "array_to_string(\"a0\",',')")) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - useDefault ? new Object[]{"1a,a,2,abc,10.1,defabc"} : new Object[]{"null,1a,a,2,defabc"} - ) - ); - } - - @Test - public void testArrayAggMaxBytes() throws Exception - { - cannotVectorize(); - testQuery( - "SELECT ARRAY_AGG(l1, 128), ARRAY_AGG(DISTINCT l1, 128) FROM numfoo", - ImmutableList.of( - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE3) - .intervals(querySegmentSpec(Filtration.eternity())) - .granularity(Granularities.ALL) - .aggregators( - aggregators( - new ExpressionLambdaAggregatorFactory( - "a0", - ImmutableSet.of("l1"), - "__acc", - "[]", - "[]", - "array_append(\"__acc\", \"l1\")", - "array_concat(\"__acc\", \"a0\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(128), - TestExprMacroTable.INSTANCE - ), - new ExpressionLambdaAggregatorFactory( - "a1", - ImmutableSet.of("l1"), - "__acc", - "[]", - "[]", - "array_set_add(\"__acc\", \"l1\")", - "array_set_add_all(\"__acc\", \"a1\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(128), - TestExprMacroTable.INSTANCE - ) - ) - ) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - useDefault - ? new Object[]{"[7,325323,0,0,0,0]", "[0,7,325323]"} - : new Object[]{"[7,325323,0,null,null,null]", "[0,null,7,325323]"} - ) - ); - } - - @Test - public void testArrayAggAsArrayFromJoin() throws Exception - { - cannotVectorize(); - List expectedResults; - if (useDefault) { - expectedResults = ImmutableList.of( - new Object[]{"a", "[\"2\",\"10.1\"]", "2,10.1"}, - new Object[]{"a", "[\"2\",\"10.1\"]", "2,10.1"}, - new Object[]{"a", "[\"2\",\"10.1\"]", "2,10.1"}, - new Object[]{"b", "[\"1\",\"abc\",\"def\"]", "1,abc,def"}, - new Object[]{"b", "[\"1\",\"abc\",\"def\"]", "1,abc,def"}, - new Object[]{"b", "[\"1\",\"abc\",\"def\"]", "1,abc,def"} - ); - } else { - expectedResults = ImmutableList.of( - new Object[]{"a", "[\"\",\"2\",\"10.1\"]", ",2,10.1"}, - new Object[]{"a", "[\"\",\"2\",\"10.1\"]", ",2,10.1"}, - new Object[]{"a", "[\"\",\"2\",\"10.1\"]", ",2,10.1"}, - new Object[]{"b", "[\"1\",\"abc\",\"def\"]", "1,abc,def"}, - new Object[]{"b", "[\"1\",\"abc\",\"def\"]", "1,abc,def"}, - new Object[]{"b", "[\"1\",\"abc\",\"def\"]", "1,abc,def"} - ); - } - testQuery( - "SELECT numfoo.dim4, j.arr, ARRAY_TO_STRING(j.arr, ',') FROM numfoo INNER JOIN (SELECT dim4, ARRAY_AGG(DISTINCT dim1) as arr FROM numfoo WHERE dim1 is not null GROUP BY 1) as j ON numfoo.dim4 = j.dim4", - ImmutableList.of( - Druids.newScanQueryBuilder() - .dataSource( - join( - new TableDataSource(CalciteTests.DATASOURCE3), - new QueryDataSource( - GroupByQuery.builder() - .setDataSource(CalciteTests.DATASOURCE3) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setDimFilter(not(selector("dim1", null, null))) - .setDimensions(new DefaultDimensionSpec("dim4", "_d0")) - .setAggregatorSpecs( - aggregators( - new ExpressionLambdaAggregatorFactory( - "a0", - ImmutableSet.of("dim1"), - "__acc", - "[]", - "[]", - "array_set_add(\"__acc\", \"dim1\")", - "array_set_add_all(\"__acc\", \"a0\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(1024), - TestExprMacroTable.INSTANCE - ) - ) - ) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - "j0.", - "(\"dim4\" == \"j0._d0\")", - JoinType.INNER, - null - ) - ) - .virtualColumns( - expressionVirtualColumn("v0", "array_to_string(\"j0.a0\",',')", ValueType.STRING) - ) - .intervals(querySegmentSpec(Filtration.eternity())) - .columns("dim4", "j0.a0", "v0") - .context(QUERY_CONTEXT_DEFAULT) - .resultFormat(ResultFormat.RESULT_FORMAT_COMPACTED_LIST) - .legacy(false) - .build() - - ), - expectedResults - ); - } - - @Test - public void testArrayAggGroupByArrayAggFromSubquery() throws Exception - { - cannotVectorize(); - // yo, can't group on array types right now so expect failure - expectedException.expect(RuntimeException.class); - expectedException.expectMessage("Cannot create query type helper from invalid type [STRING_ARRAY]"); - testQuery( - "SELECT dim2, arr, COUNT(*) FROM (SELECT dim2, ARRAY_AGG(DISTINCT dim1) as arr FROM foo WHERE dim1 is not null GROUP BY 1 LIMIT 5) GROUP BY 1,2", - ImmutableList.of(), - ImmutableList.of() - ); - } - - @Test - public void testArrayAggArrayContainsSubquery() throws Exception - { - cannotVectorize(); - List expectedResults; - if (useDefault) { - expectedResults = ImmutableList.of( - new Object[]{"10.1", ""}, - new Object[]{"2", ""}, - new Object[]{"1", "a"}, - new Object[]{"def", "abc"}, - new Object[]{"abc", ""} - ); - } else { - expectedResults = ImmutableList.of( - new Object[]{"", "a"}, - new Object[]{"10.1", null}, - new Object[]{"2", ""}, - new Object[]{"1", "a"}, - new Object[]{"def", "abc"}, - new Object[]{"abc", null} - ); - } - testQuery( - "SELECT dim1,dim2 FROM foo WHERE ARRAY_CONTAINS((SELECT ARRAY_AGG(DISTINCT dim1) FROM foo WHERE dim1 is not null), dim1)", - ImmutableList.of( - Druids.newScanQueryBuilder() - .dataSource( - join( - new TableDataSource(CalciteTests.DATASOURCE1), - new QueryDataSource( - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(querySegmentSpec(Filtration.eternity())) - .granularity(Granularities.ALL) - .filters(not(selector("dim1", null, null))) - .aggregators( - aggregators( - new ExpressionLambdaAggregatorFactory( - "a0", - ImmutableSet.of("dim1"), - "__acc", - "[]", - "[]", - "array_set_add(\"__acc\", \"dim1\")", - "array_set_add_all(\"__acc\", \"a0\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(1024), - TestExprMacroTable.INSTANCE - ) - ) - ) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ), - "j0.", - "1", - JoinType.LEFT, - null - ) - ) - .intervals(querySegmentSpec(Filtration.eternity())) - .filters( - new ExpressionDimFilter( - "array_contains(\"j0.a0\",\"dim1\")", - TestExprMacroTable.INSTANCE - ) - ) - .columns("dim1", "dim2") - .context(QUERY_CONTEXT_DEFAULT) - .resultFormat(ResultFormat.RESULT_FORMAT_COMPACTED_LIST) - .legacy(false) - .build() - - ), - expectedResults - ); - } - - @Test - public void testArrayAggGroupByArrayContainsSubquery() throws Exception - { - cannotVectorize(); - List expectedResults; - if (useDefault) { - expectedResults = ImmutableList.of( - new Object[]{"", 3L}, - new Object[]{"a", 1L}, - new Object[]{"abc", 1L} - ); - } else { - expectedResults = ImmutableList.of( - new Object[]{null, 2L}, - new Object[]{"", 1L}, - new Object[]{"a", 2L}, - new Object[]{"abc", 1L} - ); - } - testQuery( - "SELECT dim2, COUNT(*) FROM foo WHERE ARRAY_CONTAINS((SELECT ARRAY_AGG(DISTINCT dim1) FROM foo WHERE dim1 is not null), dim1) GROUP BY 1", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource( - join( - new TableDataSource(CalciteTests.DATASOURCE1), - new QueryDataSource( - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(querySegmentSpec(Filtration.eternity())) - .granularity(Granularities.ALL) - .filters(not(selector("dim1", null, null))) - .aggregators( - aggregators( - new ExpressionLambdaAggregatorFactory( - "a0", - ImmutableSet.of("dim1"), - "__acc", - "[]", - "[]", - "array_set_add(\"__acc\", \"dim1\")", - "array_set_add_all(\"__acc\", \"a0\")", - null, - "if(array_length(o) == 0, null, o)", - new HumanReadableBytes(1024), - TestExprMacroTable.INSTANCE - ) - ) - ) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ), - "j0.", - "1", - JoinType.LEFT, - null - ) - ) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setDimFilter( - new ExpressionDimFilter( - "array_contains(\"j0.a0\",\"dim1\")", - TestExprMacroTable.INSTANCE - ) - ) - .setDimensions(dimensions(new DefaultDimensionSpec("dim2", "d0"))) - .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))) - .setGranularity(Granularities.ALL) - .setLimitSpec(NoopLimitSpec.instance()) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - - ), - expectedResults - ); - } - @Test public void testCountAndAverageByConstantVirtualColumn() throws Exception {