Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

array handling improvements (#11233) 

* fix jdbc array handling, split handling for some array and multi value operator, split and add more tests

* formatting
This commit is contained in:
Clint Wylie 2021-05-13 18:50:32 -07:00 committed by GitHub
parent 4100c5edc0
commit 3649c608d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 3429 additions and 1848 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
core/src/main/java/org/apache/druid/math/expr/ExprEval.java
View File

@ -1303,6 +1303,9 @@ public abstract class ExprEval<T>
return null;
}
return Arrays.stream(value).map(value -> {
if (value == null) {
return null;
}
Long lv = GuavaUtils.tryParseLong(value);
if (lv == null) {
Double d = Doubles.tryParse(value);
@ -1320,7 +1323,12 @@ public abstract class ExprEval<T>
if (value == null) {
return null;
}
return Arrays.stream(value).map(Doubles::tryParse).toArray(Double[]::new);
return Arrays.stream(value).map(val -> {
if (val == null) {
return null;
}
return Doubles.tryParse(val);
}).toArray(Double[]::new);
}
}
}

14
core/src/test/java/org/apache/druid/math/expr/ExprEvalTest.java
View File

@ -259,6 +259,20 @@ public class ExprEvalTest extends InitializedNullHandlingTest
);
}
@Test
public void testStringArrayToNumberArray()
{
ExprEval someStringArray = ExprEval.ofStringArray(new String[]{"1", "2", "foo", null, "3.3"});
Assert.assertArrayEquals(
new Long[]{1L, 2L, null, null, 3L},
someStringArray.asLongArray()
);
Assert.assertArrayEquals(
new Double[]{1.0, 2.0, null, null, 3.3},
someStringArray.asDoubleArray()
);
}
private void assertExpr(int position, Object expected)
{
assertExpr(position, ExprEval.bestEffortOf(expected));

1
docs/querying/sql.md
View File

@ -1022,6 +1022,7 @@ Connection context can be specified as JDBC connection properties or as a "conte
|---------|-----------|-------------|
|`sqlQueryId`|Unique identifier given to this SQL query. For HTTP client, it will be returned in `X-Druid-SQL-Query-Id` header.|auto-generated|
|`sqlTimeZone`|Sets the time zone for this connection, which will affect how time functions and timestamp literals behave. Should be a time zone name like "America/Los_Angeles" or offset like "-08:00".|druid.sql.planner.sqlTimeZone on the Broker (default: UTC)|
|`sqlStringifyArrays`|When set to true, result columns which return array values will be serialized into a JSON string in the response instead of as an array (default: true, except for JDBC connections, where it is always false)|
|`useApproximateCountDistinct`|Whether to use an approximate cardinality algorithm for `COUNT(DISTINCT foo)`.|druid.sql.planner.useApproximateCountDistinct on the Broker (default: true)|
|`useGroupingSetForExactDistinct`|Whether to use grouping sets to execute queries with multiple exact distinct aggregations.|druid.sql.planner.useGroupingSetForExactDistinct on the Broker (default: false)|
|`useApproximateTopN`|Whether to use approximate [TopN queries](topnquery.md) when a SQL query could be expressed as such. If false, exact [GroupBy queries](groupbyquery.md) will be used instead.|druid.sql.planner.useApproximateTopN on the Broker (default: true)|

3
sql/src/main/java/org/apache/druid/sql/avatica/DruidMeta.java
View File

@ -45,6 +45,7 @@ import org.apache.druid.server.security.AuthenticatorMapper;
import org.apache.druid.server.security.ForbiddenException;
import org.apache.druid.sql.SqlLifecycleFactory;
import org.apache.druid.sql.calcite.planner.Calcites;
import org.apache.druid.sql.calcite.planner.PlannerContext;
import org.joda.time.Interval;
import javax.annotation.Nonnull;
@ -109,6 +110,8 @@ public class DruidMeta extends MetaImpl
context.put(entry);
}
}
// we don't want to stringify arrays for JDBC ever because avatica needs to handle this
context.put(PlannerContext.CTX_SQL_STRINGIFY_ARRAYS, false);
openDruidConnection(ch.id, context.build());
}

30
sql/src/main/java/org/apache/druid/sql/avatica/DruidStatement.java
View File

@ -28,6 +28,7 @@ import org.apache.calcite.avatica.Meta;
import org.apache.calcite.avatica.remote.TypedValue;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.concurrent.Execs;
@ -114,12 +115,29 @@ public class DruidStatement implements Closeable
for (int i = 0; i < fieldList.size(); i++) {
RelDataTypeField field = fieldList.get(i);
final ColumnMetaData.Rep rep = QueryMaker.rep(field.getType().getSqlTypeName());
final ColumnMetaData.ScalarType columnType = ColumnMetaData.scalar(
field.getType().getSqlTypeName().getJdbcOrdinal(),
field.getType().getSqlTypeName().getName(),
rep
);
final ColumnMetaData.AvaticaType columnType;
if (field.getType().getSqlTypeName() == SqlTypeName.ARRAY) {
final ColumnMetaData.Rep elementRep = QueryMaker.rep(field.getType().getComponentType().getSqlTypeName());
final ColumnMetaData.ScalarType elementType = ColumnMetaData.scalar(
field.getType().getComponentType().getSqlTypeName().getJdbcOrdinal(),
field.getType().getComponentType().getSqlTypeName().getName(),
elementRep
);
final ColumnMetaData.Rep arrayRep = QueryMaker.rep(field.getType().getSqlTypeName());
columnType = ColumnMetaData.array(
elementType,
field.getType().getSqlTypeName().getName(),
arrayRep
);
} else {
final ColumnMetaData.Rep rep = QueryMaker.rep(field.getType().getSqlTypeName());
columnType = ColumnMetaData.scalar(
field.getType().getSqlTypeName().getJdbcOrdinal(),
field.getType().getSqlTypeName().getName(),
rep
);
}
columns.add(
new ColumnMetaData(
i, // ordinal

23
sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java
View File

@ -257,7 +257,7 @@ public class OperatorConversions
/**
* Sets the return type of the operator to "typeName", marked as non-nullable.
*
* One of {@link #returnTypeNonNull}, {@link #returnTypeNullable}, or
* One of {@link #returnTypeNonNull}, {@link #returnTypeNullable}, {@link #returnTypeNullableArray}, or
* {@link #returnTypeInference(SqlReturnTypeInference)} must be used before calling {@link #build()}. These methods
* cannot be mixed; you must call exactly one.
*/
@ -274,7 +274,7 @@ public class OperatorConversions
/**
* Sets the return type of the operator to "typeName", marked as nullable.
*
* One of {@link #returnTypeNonNull}, {@link #returnTypeNullable}, or
* One of {@link #returnTypeNonNull}, {@link #returnTypeNullable}, {@link #returnTypeNullableArray}, or
* {@link #returnTypeInference(SqlReturnTypeInference)} must be used before calling {@link #build()}. These methods
* cannot be mixed; you must call exactly one.
*/
@ -287,11 +287,28 @@ public class OperatorConversions
);
return this;
}
/**
* Sets the return type of the operator to an array type with elements of "typeName", marked as nullable.
*
* One of {@link #returnTypeNonNull}, {@link #returnTypeNullable}, {@link #returnTypeNullableArray}, or
* {@link #returnTypeInference(SqlReturnTypeInference)} must be used before calling {@link #build()}. These methods
* cannot be mixed; you must call exactly one.
*/
public OperatorBuilder returnTypeNullableArray(final SqlTypeName elementTypeName)
{
Preconditions.checkState(this.returnTypeInference == null, "Cannot set return type multiple times");
this.returnTypeInference = ReturnTypes.explicit(
factory -> Calcites.createSqlArrayTypeWithNullability(factory, elementTypeName, true)
);
return this;
}
/**
* Provides customized return type inference logic.
*
* One of {@link #returnTypeNonNull}, {@link #returnTypeNullable}, or
* One of {@link #returnTypeNonNull}, {@link #returnTypeNullable}, {@link #returnTypeNullableArray}, or
* {@link #returnTypeInference(SqlReturnTypeInference)} must be used before calling {@link #build()}. These methods
* cannot be mixed; you must call exactly one.
*/

13
sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringAppendOperatorConversion.java → sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayAppendOperatorConversion.java
View File

@ -24,22 +24,25 @@ import org.apache.calcite.sql.SqlFunction;
import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.type.OperandTypes;
import org.apache.calcite.sql.type.ReturnTypes;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.sql.calcite.expression.DruidExpression;
import org.apache.druid.sql.calcite.expression.OperatorConversions;
import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
import org.apache.druid.sql.calcite.planner.PlannerContext;
public class MultiValueStringAppendOperatorConversion implements SqlOperatorConversion
public class ArrayAppendOperatorConversion implements SqlOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder("MV_APPEND")
.operatorBuilder("ARRAY_APPEND")
.operandTypeChecker(
OperandTypes.sequence(
"(array,expr)",
OperandTypes.family(SqlTypeFamily.STRING),
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING)
),
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.STRING),
OperandTypes.family(SqlTypeFamily.NUMERIC)
@ -47,7 +50,7 @@ public class MultiValueStringAppendOperatorConversion implements SqlOperatorConv
)
)
.functionCategory(SqlFunctionCategory.STRING)
.returnTypeNonNull(SqlTypeName.VARCHAR)
.returnTypeInference(ReturnTypes.ARG0_NULLABLE)
.build();
@Override

18
sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringConcatOperatorConversion.java → sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayConcatOperatorConversion.java
View File

@ -24,27 +24,33 @@ import org.apache.calcite.sql.SqlFunction;
import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.type.OperandTypes;
import org.apache.calcite.sql.type.ReturnTypes;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.sql.calcite.expression.DruidExpression;
import org.apache.druid.sql.calcite.expression.OperatorConversions;
import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
import org.apache.druid.sql.calcite.planner.PlannerContext;
public class MultiValueStringConcatOperatorConversion implements SqlOperatorConversion
public class ArrayConcatOperatorConversion implements SqlOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder("MV_CONCAT")
.operatorBuilder("ARRAY_CONCAT")
.operandTypeChecker(
OperandTypes.sequence(
"(array,array)",
OperandTypes.family(SqlTypeFamily.STRING),
OperandTypes.family(SqlTypeFamily.STRING)
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING)
),
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING)
)
)
)
.functionCategory(SqlFunctionCategory.STRING)
.returnTypeNonNull(SqlTypeName.VARCHAR)
.returnTypeInference(ReturnTypes.ARG0_NULLABLE)
.build();
@Override

3
sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayContainsOperatorConversion.java
View File

@ -57,7 +57,8 @@ public class ArrayContainsOperatorConversion extends BaseExpressionDimFilterOper
),
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING)
OperandTypes.family(SqlTypeFamily.STRING),
OperandTypes.family(SqlTypeFamily.NUMERIC)
)
)
)

3
sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayLengthOperatorConversion.java
View File

@ -38,9 +38,8 @@ public class ArrayLengthOperatorConversion implements SqlOperatorConversion
.operatorBuilder("ARRAY_LENGTH")
.operandTypeChecker(
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.STRING),
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.MULTISET)
OperandTypes.family(SqlTypeFamily.STRING)
)
)
.functionCategory(SqlFunctionCategory.STRING)

3
sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOffsetOfOperatorConversion.java
View File

@ -40,9 +40,8 @@ public class ArrayOffsetOfOperatorConversion implements SqlOperatorConversion
OperandTypes.sequence(
"(array,expr)",
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.STRING),
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.MULTISET)
OperandTypes.family(SqlTypeFamily.STRING)
),
OperandTypes.family(SqlTypeFamily.ANY)
)

12
sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOffsetOperatorConversion.java
View File

@ -25,7 +25,6 @@ import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.type.OperandTypes;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.sql.calcite.expression.DruidExpression;
import org.apache.druid.sql.calcite.expression.OperatorConversions;
@ -39,16 +38,15 @@ public class ArrayOffsetOperatorConversion implements SqlOperatorConversion
.operandTypeChecker(
OperandTypes.sequence(
"(array,expr)",
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.STRING),
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.MULTISET)
),
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING)
),
OperandTypes.family(SqlTypeFamily.NUMERIC)
)
)
.functionCategory(SqlFunctionCategory.STRING)
.returnTypeNonNull(SqlTypeName.VARCHAR)
.returnTypeInference(ArrayOrdinalOperatorConversion.ARG0_ELEMENT_INFERENCE)
.build();
@Override

3
sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOrdinalOfOperatorConversion.java
View File

@ -40,9 +40,8 @@ public class ArrayOrdinalOfOperatorConversion implements SqlOperatorConversion
OperandTypes.sequence(
"(array,expr)",
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.STRING),
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.MULTISET)
OperandTypes.family(SqlTypeFamily.STRING)
),
OperandTypes.family(SqlTypeFamily.ANY)
)

25
sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOrdinalOperatorConversion.java
View File

@ -19,13 +19,16 @@
package org.apache.druid.sql.calcite.expression.builtin;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlFunction;
import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.SqlOperatorBinding;
import org.apache.calcite.sql.type.OperandTypes;
import org.apache.calcite.sql.type.SqlReturnTypeInference;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.type.SqlTypeUtil;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.sql.calcite.expression.DruidExpression;
import org.apache.druid.sql.calcite.expression.OperatorConversions;
@ -34,21 +37,22 @@ import org.apache.druid.sql.calcite.planner.PlannerContext;
public class ArrayOrdinalOperatorConversion implements SqlOperatorConversion
{
static final SqlReturnTypeInference ARG0_ELEMENT_INFERENCE = new ArrayElementReturnTypeInference();
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder("ARRAY_ORDINAL")
.operandTypeChecker(
OperandTypes.sequence(
"(array,expr)",
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.STRING),
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.MULTISET)
OperandTypes.family(SqlTypeFamily.STRING)
),
OperandTypes.family(SqlTypeFamily.NUMERIC)
)
)
.functionCategory(SqlFunctionCategory.STRING)
.returnTypeNonNull(SqlTypeName.VARCHAR)
.returnTypeInference(ARG0_ELEMENT_INFERENCE)
.build();
@Override
@ -74,4 +78,17 @@ public class ArrayOrdinalOperatorConversion implements SqlOperatorConversion
)
);
}
static class ArrayElementReturnTypeInference implements SqlReturnTypeInference
{
@Override
public RelDataType inferReturnType(SqlOperatorBinding sqlOperatorBinding)
{
RelDataType type = sqlOperatorBinding.getOperandType(0);
if (SqlTypeUtil.isArray(type)) {
type.getComponentType();
}
return type;
}
}
}

13
sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringPrependOperatorConversion.java → sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayPrependOperatorConversion.java
View File

@ -24,18 +24,18 @@ import org.apache.calcite.sql.SqlFunction;
import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.type.OperandTypes;
import org.apache.calcite.sql.type.ReturnTypes;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.sql.calcite.expression.DruidExpression;
import org.apache.druid.sql.calcite.expression.OperatorConversions;
import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
import org.apache.druid.sql.calcite.planner.PlannerContext;
public class MultiValueStringPrependOperatorConversion implements SqlOperatorConversion
public class ArrayPrependOperatorConversion implements SqlOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder("MV_PREPEND")
.operatorBuilder("ARRAY_PREPEND")
.operandTypeChecker(
OperandTypes.sequence(
"(expr,array)",
@ -43,11 +43,14 @@ public class MultiValueStringPrependOperatorConversion implements SqlOperatorCon
OperandTypes.family(SqlTypeFamily.STRING),
OperandTypes.family(SqlTypeFamily.NUMERIC)
),
OperandTypes.family(SqlTypeFamily.STRING)
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING)
)
)
)
.functionCategory(SqlFunctionCategory.STRING)
.returnTypeNonNull(SqlTypeName.VARCHAR)
.returnTypeInference(ReturnTypes.ARG1_NULLABLE)
.build();
@Override

18
sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringSliceOperatorConversion.java → sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArraySliceOperatorConversion.java
View File

@ -24,35 +24,41 @@ import org.apache.calcite.sql.SqlFunction;
import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.type.OperandTypes;
import org.apache.calcite.sql.type.ReturnTypes;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.sql.calcite.expression.DruidExpression;
import org.apache.druid.sql.calcite.expression.OperatorConversions;
import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
import org.apache.druid.sql.calcite.planner.PlannerContext;
public class MultiValueStringSliceOperatorConversion implements SqlOperatorConversion
public class ArraySliceOperatorConversion implements SqlOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder("MV_SLICE")
.operatorBuilder("ARRAY_SLICE")
.operandTypeChecker(
OperandTypes.or(
OperandTypes.sequence(
"(expr,start)",
OperandTypes.family(SqlTypeFamily.STRING),
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING)
),
OperandTypes.family(SqlTypeFamily.NUMERIC)
),
OperandTypes.sequence(
"(expr,start,end)",
OperandTypes.family(SqlTypeFamily.STRING),
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING)
),
OperandTypes.family(SqlTypeFamily.NUMERIC),
OperandTypes.family(SqlTypeFamily.NUMERIC)
)
)
)
.functionCategory(SqlFunctionCategory.STRING)
.returnTypeNonNull(SqlTypeName.VARCHAR)
.returnTypeInference(ReturnTypes.ARG0_NULLABLE)
.build();
@Override

3
sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayToStringOperatorConversion.java
View File

@ -40,9 +40,8 @@ public class ArrayToStringOperatorConversion implements SqlOperatorConversion
OperandTypes.sequence(
"(array,expr)",
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.STRING),
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.MULTISET)
OperandTypes.family(SqlTypeFamily.STRING)
),
OperandTypes.family(SqlTypeFamily.ANY)
)

297
sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringOperatorConversions.java Normal file
View File

@ -0,0 +1,297 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.sql.calcite.expression.builtin;
import org.apache.calcite.sql.SqlFunction;
import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.type.OperandTypes;
import org.apache.calcite.sql.type.ReturnTypes;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.druid.sql.calcite.expression.AliasedOperatorConversion;
import org.apache.druid.sql.calcite.expression.OperatorConversions;
/**
* Array functions which return an array, but are used in a multi-valued string dimension context instead will output
* {@link SqlTypeName#VARCHAR} instead of {@link SqlTypeName#ARRAY}. On the backend, these functions are identical,
* so these classes only override the signature information.
*/
public class MultiValueStringOperatorConversions
{
public static class Append extends ArrayAppendOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder("MV_APPEND")
.operandTypeChecker(
OperandTypes.sequence(
"(array,expr)",
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING)
),
OperandTypes.family(SqlTypeFamily.STRING)
)
)
.functionCategory(SqlFunctionCategory.STRING)
.returnTypeNullable(SqlTypeName.VARCHAR)
.build();
@Override
public SqlOperator calciteOperator()
{
return SQL_FUNCTION;
}
}
public static class Prepend extends ArrayPrependOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder("MV_PREPEND")
.operandTypeChecker(
OperandTypes.sequence(
"(expr,array)",
OperandTypes.family(SqlTypeFamily.STRING),
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING)
)
)
)
.functionCategory(SqlFunctionCategory.STRING)
.returnTypeNullable(SqlTypeName.VARCHAR)
.build();
@Override
public SqlOperator calciteOperator()
{
return SQL_FUNCTION;
}
}
public static class Concat extends ArrayConcatOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder("MV_CONCAT")
.operandTypeChecker(
OperandTypes.sequence(
"(array,array)",
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING)
),
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING)
)
)
)
.functionCategory(SqlFunctionCategory.STRING)
.returnTypeNullable(SqlTypeName.VARCHAR)
.build();
@Override
public SqlOperator calciteOperator()
{
return SQL_FUNCTION;
}
}
public static class Contains extends ArrayContainsOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder("MV_CONTAINS")
.operandTypeChecker(
OperandTypes.sequence(
"(array,array)",
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING)
),
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING),
OperandTypes.family(SqlTypeFamily.NUMERIC)
)
)
)
.returnTypeInference(ReturnTypes.BOOLEAN)
.build();
@Override
public SqlOperator calciteOperator()
{
return SQL_FUNCTION;
}
}
public static class Offset extends ArrayOffsetOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder("MV_OFFSET")
.operandTypeChecker(
OperandTypes.sequence(
"(array,expr)",
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING)
),
OperandTypes.family(SqlTypeFamily.NUMERIC)
)
)
.functionCategory(SqlFunctionCategory.STRING)
.returnTypeNullable(SqlTypeName.VARCHAR)
.build();
@Override
public SqlOperator calciteOperator()
{
return SQL_FUNCTION;
}
}
public static class Ordinal extends ArrayOrdinalOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder("MV_ORDINAL")
.operandTypeChecker(
OperandTypes.sequence(
"(array,expr)",
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING)
),
OperandTypes.family(SqlTypeFamily.NUMERIC)
)
)
.functionCategory(SqlFunctionCategory.STRING)
.returnTypeNullable(SqlTypeName.VARCHAR)
.build();
@Override
public SqlOperator calciteOperator()
{
return SQL_FUNCTION;
}
}
public static class Slice extends ArraySliceOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder("MV_SLICE")
.operandTypeChecker(
OperandTypes.or(
OperandTypes.sequence(
"(expr,start)",
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING)
),
OperandTypes.family(SqlTypeFamily.NUMERIC)
),
OperandTypes.sequence(
"(expr,start,end)",
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.ARRAY),
OperandTypes.family(SqlTypeFamily.STRING)
),
OperandTypes.family(SqlTypeFamily.NUMERIC),
OperandTypes.family(SqlTypeFamily.NUMERIC)
)
)
)
.functionCategory(SqlFunctionCategory.STRING)
.returnTypeNullable(SqlTypeName.VARCHAR)
.build();
@Override
public SqlOperator calciteOperator()
{
return SQL_FUNCTION;
}
}
public static class StringToMultiString extends StringToArrayOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder("STRING_TO_MV")
.operandTypeChecker(
OperandTypes.sequence(
"(string,expr)",
OperandTypes.family(SqlTypeFamily.STRING),
OperandTypes.family(SqlTypeFamily.STRING)
)
)
.functionCategory(SqlFunctionCategory.STRING)
.returnTypeNullable(SqlTypeName.VARCHAR)
.build();
@Override
public SqlOperator calciteOperator()
{
return SQL_FUNCTION;
}
}
public static class MultiStringToString extends AliasedOperatorConversion
{
public MultiStringToString()
{
super(new ArrayToStringOperatorConversion(), "MV_TO_STRING");
}
}
public static class Length extends AliasedOperatorConversion
{
public Length()
{
super(new ArrayLengthOperatorConversion(), "MV_LENGTH");
}
}
public static class OffsetOf extends AliasedOperatorConversion
{
public OffsetOf()
{
super(new ArrayOffsetOfOperatorConversion(), "MV_OFFSET_OF");
}
}
public static class OrdinalOf extends AliasedOperatorConversion
{
public OrdinalOf()
{
super(new ArrayOrdinalOfOperatorConversion(), "MV_ORDINAL_OF");
}
}
public static class Overlap extends AliasedOperatorConversion
{
public Overlap()
{
super(new ArrayOverlapOperatorConversion(), "MV_OVERLAP");
}
}
private MultiValueStringOperatorConversions()
{
// no instantiation
}
}

6
sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/StringToMultiValueStringOperatorConversion.java → sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/StringToArrayOperatorConversion.java
View File

@ -32,11 +32,11 @@ import org.apache.druid.sql.calcite.expression.OperatorConversions;
import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
import org.apache.druid.sql.calcite.planner.PlannerContext;
public class StringToMultiValueStringOperatorConversion implements SqlOperatorConversion
public class StringToArrayOperatorConversion implements SqlOperatorConversion
{
// note: since this function produces an array
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder("STRING_TO_MV")
.operatorBuilder("STRING_TO_ARRAY")
.operandTypeChecker(
OperandTypes.sequence(
"(string,expr)",
@ -45,7 +45,7 @@ public class StringToMultiValueStringOperatorConversion implements SqlOperatorCo
)
)
.functionCategory(SqlFunctionCategory.STRING)
.returnTypeNonNull(SqlTypeName.VARCHAR)
.returnTypeNullableArray(SqlTypeName.VARCHAR)
.build();
@Override

358
sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java
View File

@ -51,6 +51,8 @@ import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
import org.apache.druid.sql.calcite.expression.UnaryFunctionOperatorConversion;
import org.apache.druid.sql.calcite.expression.UnaryPrefixOperatorConversion;
import org.apache.druid.sql.calcite.expression.UnarySuffixOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.ArrayAppendOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.ArrayConcatOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.ArrayConstructorOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.ArrayContainsOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.ArrayLengthOperatorConversion;
@ -59,6 +61,8 @@ import org.apache.druid.sql.calcite.expression.builtin.ArrayOffsetOperatorConver
import org.apache.druid.sql.calcite.expression.builtin.ArrayOrdinalOfOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.ArrayOrdinalOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.ArrayOverlapOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.ArrayPrependOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.ArraySliceOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.ArrayToStringOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.BTrimOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.CastOperatorConversion;
@ -78,10 +82,7 @@ import org.apache.druid.sql.calcite.expression.builtin.LeastOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.LeftOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.LikeOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.MillisToTimestampOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.MultiValueStringAppendOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.MultiValueStringConcatOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.MultiValueStringPrependOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.MultiValueStringSliceOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.MultiValueStringOperatorConversions;
import org.apache.druid.sql.calcite.expression.builtin.ParseLongOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.PositionOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.RPadOperatorConversion;
@ -94,7 +95,7 @@ import org.apache.druid.sql.calcite.expression.builtin.ReverseOperatorConversion
import org.apache.druid.sql.calcite.expression.builtin.RightOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.RoundOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.StringFormatOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.StringToMultiValueStringOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.StringToArrayOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.StrposOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.SubstringOperatorConversion;
import org.apache.druid.sql.calcite.expression.builtin.TextcatOperatorConversion;
@ -123,19 +124,19 @@ public class DruidOperatorTable implements SqlOperatorTable
{
private static final List<SqlAggregator> STANDARD_AGGREGATORS =
ImmutableList.<SqlAggregator>builder()
.add(new ApproxCountDistinctSqlAggregator())
.add(new AvgSqlAggregator())
.add(new CountSqlAggregator())
.add(EarliestLatestAnySqlAggregator.EARLIEST)
.add(EarliestLatestAnySqlAggregator.LATEST)
.add(EarliestLatestAnySqlAggregator.ANY_VALUE)
.add(new MinSqlAggregator())
.add(new MaxSqlAggregator())
.add(new SumSqlAggregator())
.add(new SumZeroSqlAggregator())
.add(new GroupingSqlAggregator())
.add(new ArraySqlAggregator())
.build();
.add(new ApproxCountDistinctSqlAggregator())
.add(new AvgSqlAggregator())
.add(new CountSqlAggregator())
.add(EarliestLatestAnySqlAggregator.EARLIEST)
.add(EarliestLatestAnySqlAggregator.LATEST)
.add(EarliestLatestAnySqlAggregator.ANY_VALUE)
.add(new MinSqlAggregator())
.add(new MaxSqlAggregator())
.add(new SumSqlAggregator())
.add(new SumZeroSqlAggregator())
.add(new GroupingSqlAggregator())
.add(new ArraySqlAggregator())
.build();
// STRLEN has so many aliases.
@ -146,182 +147,199 @@ public class DruidOperatorTable implements SqlOperatorTable
private static final List<SqlOperatorConversion> TIME_OPERATOR_CONVERSIONS =
ImmutableList.<SqlOperatorConversion>builder()
.add(new CeilOperatorConversion())
.add(new DateTruncOperatorConversion())
.add(new ExtractOperatorConversion())
.add(new FloorOperatorConversion())
.add(new MillisToTimestampOperatorConversion())
.add(new TimeArithmeticOperatorConversion.TimeMinusIntervalOperatorConversion())
.add(new TimeArithmeticOperatorConversion.TimePlusIntervalOperatorConversion())
.add(new TimeExtractOperatorConversion())
.add(new TimeCeilOperatorConversion())
.add(new TimeFloorOperatorConversion())
.add(new TimeFormatOperatorConversion())
.add(new TimeParseOperatorConversion())
.add(new TimeShiftOperatorConversion())
.add(new TimestampToMillisOperatorConversion())
.build();
.add(new CeilOperatorConversion())
.add(new DateTruncOperatorConversion())
.add(new ExtractOperatorConversion())
.add(new FloorOperatorConversion())
.add(new MillisToTimestampOperatorConversion())
.add(new TimeArithmeticOperatorConversion.TimeMinusIntervalOperatorConversion())
.add(new TimeArithmeticOperatorConversion.TimePlusIntervalOperatorConversion())
.add(new TimeExtractOperatorConversion())
.add(new TimeCeilOperatorConversion())
.add(new TimeFloorOperatorConversion())
.add(new TimeFormatOperatorConversion())
.add(new TimeParseOperatorConversion())
.add(new TimeShiftOperatorConversion())
.add(new TimestampToMillisOperatorConversion())
.build();
private static final List<SqlOperatorConversion> STRING_OPERATOR_CONVERSIONS =
ImmutableList.<SqlOperatorConversion>builder()
.add(new BTrimOperatorConversion())
.add(new LikeOperatorConversion())
.add(new LTrimOperatorConversion())
.add(new PositionOperatorConversion())
.add(new RegexpExtractOperatorConversion())
.add(new RegexpLikeOperatorConversion())
.add(new RTrimOperatorConversion())
.add(new ParseLongOperatorConversion())
.add(new StringFormatOperatorConversion())
.add(new StrposOperatorConversion())
.add(new SubstringOperatorConversion())
.add(new RightOperatorConversion())
.add(new LeftOperatorConversion())
.add(new ReverseOperatorConversion())
.add(new RepeatOperatorConversion())
.add(new AliasedOperatorConversion(new SubstringOperatorConversion(), "SUBSTR"))
.add(new ConcatOperatorConversion())
.add(new TextcatOperatorConversion())
.add(new TrimOperatorConversion())
.add(new TruncateOperatorConversion())
.add(new AliasedOperatorConversion(new TruncateOperatorConversion(), "TRUNC"))
.add(new LPadOperatorConversion())
.add(new RPadOperatorConversion())
.add(ContainsOperatorConversion.caseSensitive())
.add(ContainsOperatorConversion.caseInsensitive())
.build();
.add(new BTrimOperatorConversion())
.add(new LikeOperatorConversion())
.add(new LTrimOperatorConversion())
.add(new PositionOperatorConversion())
.add(new RegexpExtractOperatorConversion())
.add(new RegexpLikeOperatorConversion())
.add(new RTrimOperatorConversion())
.add(new ParseLongOperatorConversion())
.add(new StringFormatOperatorConversion())
.add(new StrposOperatorConversion())
.add(new SubstringOperatorConversion())
.add(new RightOperatorConversion())
.add(new LeftOperatorConversion())
.add(new ReverseOperatorConversion())
.add(new RepeatOperatorConversion())
.add(new AliasedOperatorConversion(new SubstringOperatorConversion(), "SUBSTR"))
.add(new ConcatOperatorConversion())
.add(new TextcatOperatorConversion())
.add(new TrimOperatorConversion())
.add(new TruncateOperatorConversion())
.add(new AliasedOperatorConversion(new TruncateOperatorConversion(), "TRUNC"))
.add(new LPadOperatorConversion())
.add(new RPadOperatorConversion())
.add(ContainsOperatorConversion.caseSensitive())
.add(ContainsOperatorConversion.caseInsensitive())
.build();
private static final List<SqlOperatorConversion> VALUE_COERCION_OPERATOR_CONVERSIONS =
ImmutableList.<SqlOperatorConversion>builder()
.add(new CastOperatorConversion())
.add(new ReinterpretOperatorConversion())
.build();
.add(new CastOperatorConversion())
.add(new ReinterpretOperatorConversion())
.build();
private static final List<SqlOperatorConversion> ARRAY_OPERATOR_CONVERSIONS =
ImmutableList.<SqlOperatorConversion>builder()
.add(new ArrayConstructorOperatorConversion())
.add(new ArrayContainsOperatorConversion())
.add(new ArrayOverlapOperatorConversion())
.add(new AliasedOperatorConversion(new ArrayContainsOperatorConversion(), "MV_CONTAINS"))
.add(new AliasedOperatorConversion(new ArrayOverlapOperatorConversion(), "MV_OVERLAP"))
.add(new ArrayLengthOperatorConversion())
.add(new AliasedOperatorConversion(new ArrayLengthOperatorConversion(), "MV_LENGTH"))
.add(new ArrayOffsetOperatorConversion())
.add(new AliasedOperatorConversion(new ArrayOffsetOperatorConversion(), "MV_OFFSET"))
.add(new ArrayOrdinalOperatorConversion())
.add(new AliasedOperatorConversion(new ArrayOrdinalOperatorConversion(), "MV_ORDINAL"))
.add(new ArrayOffsetOfOperatorConversion())
.add(new AliasedOperatorConversion(new ArrayOffsetOfOperatorConversion(), "MV_OFFSET_OF"))
.add(new ArrayOrdinalOfOperatorConversion())
.add(new AliasedOperatorConversion(new ArrayOrdinalOfOperatorConversion(), "MV_ORDINAL_OF"))
.add(new ArrayToStringOperatorConversion())
.add(new AliasedOperatorConversion(new ArrayToStringOperatorConversion(), "MV_TO_STRING"))
.build();
.add(new ArrayConstructorOperatorConversion())
.add(new ArrayContainsOperatorConversion())
.add(new ArrayConcatOperatorConversion())
.add(new ArrayOverlapOperatorConversion())
.add(new ArrayAppendOperatorConversion())
.add(new ArrayPrependOperatorConversion())
.add(new ArrayLengthOperatorConversion())
.add(new ArrayOffsetOperatorConversion())
.add(new ArrayOrdinalOperatorConversion())
.add(new ArrayOffsetOfOperatorConversion())
.add(new ArrayOrdinalOfOperatorConversion())
.add(new ArraySliceOperatorConversion())
.add(new ArrayToStringOperatorConversion())
.add(new StringToArrayOperatorConversion())
.build();
private static final List<SqlOperatorConversion> MULTIVALUE_STRING_OPERATOR_CONVERSIONS =
ImmutableList.<SqlOperatorConversion>builder()
.add(new MultiValueStringAppendOperatorConversion())
.add(new MultiValueStringConcatOperatorConversion())
.add(new MultiValueStringPrependOperatorConversion())
.add(new MultiValueStringSliceOperatorConversion())
.add(new StringToMultiValueStringOperatorConversion())
.build();
.add(new MultiValueStringOperatorConversions.Append())
.add(new MultiValueStringOperatorConversions.Prepend())
.add(new MultiValueStringOperatorConversions.Concat())
.add(new MultiValueStringOperatorConversions.Contains())
.add(new MultiValueStringOperatorConversions.Overlap())
.add(new MultiValueStringOperatorConversions.Length())
.add(new MultiValueStringOperatorConversions.Offset())
.add(new MultiValueStringOperatorConversions.Ordinal())
.add(new MultiValueStringOperatorConversions.OffsetOf())
.add(new MultiValueStringOperatorConversions.OrdinalOf())
.add(new MultiValueStringOperatorConversions.Slice())
.add(new MultiValueStringOperatorConversions.MultiStringToString())
.add(new MultiValueStringOperatorConversions.StringToMultiString())
.build();
private static final List<SqlOperatorConversion> REDUCTION_OPERATOR_CONVERSIONS =
ImmutableList.<SqlOperatorConversion>builder()
.add(new GreatestOperatorConversion())
.add(new LeastOperatorConversion())
.build();
.add(new GreatestOperatorConversion())
.add(new LeastOperatorConversion())
.build();
private static final List<SqlOperatorConversion> IPV4ADDRESS_OPERATOR_CONVERSIONS =
ImmutableList.<SqlOperatorConversion>builder()
.add(new IPv4AddressMatchOperatorConversion())
.add(new IPv4AddressParseOperatorConversion())
.add(new IPv4AddressStringifyOperatorConversion())
.build();
.add(new IPv4AddressMatchOperatorConversion())
.add(new IPv4AddressParseOperatorConversion())
.add(new IPv4AddressStringifyOperatorConversion())
.build();
private static final List<SqlOperatorConversion> BITWISE_OPERATOR_CONVERSIONS =
ImmutableList.<SqlOperatorConversion>builder()
.add(OperatorConversions.druidBinaryLongFn("BITWISE_AND", "bitwiseAnd"))
.add(OperatorConversions.druidUnaryLongFn("BITWISE_COMPLEMENT", "bitwiseComplement"))
.add(OperatorConversions.druidBinaryLongFn("BITWISE_OR", "bitwiseOr"))
.add(OperatorConversions.druidBinaryLongFn("BITWISE_SHIFT_LEFT", "bitwiseShiftLeft"))
.add(OperatorConversions.druidBinaryLongFn("BITWISE_SHIFT_RIGHT", "bitwiseShiftRight"))
.add(OperatorConversions.druidBinaryLongFn("BITWISE_XOR", "bitwiseXor"))
.add(
OperatorConversions.druidUnaryLongFn(
"BITWISE_CONVERT_DOUBLE_TO_LONG_BITS",
"bitwiseConvertDoubleToLongBits"
)
)
.add(
OperatorConversions.druidUnaryDoubleFn(
"BITWISE_CONVERT_LONG_BITS_TO_DOUBLE",
"bitwiseConvertLongBitsToDouble"
)
)
.build();
.add(OperatorConversions.druidBinaryLongFn("BITWISE_AND", "bitwiseAnd"))
.add(OperatorConversions.druidUnaryLongFn("BITWISE_COMPLEMENT", "bitwiseComplement"))
.add(OperatorConversions.druidBinaryLongFn("BITWISE_OR", "bitwiseOr"))
.add(OperatorConversions.druidBinaryLongFn("BITWISE_SHIFT_LEFT", "bitwiseShiftLeft"))
.add(OperatorConversions.druidBinaryLongFn("BITWISE_SHIFT_RIGHT", "bitwiseShiftRight"))
.add(OperatorConversions.druidBinaryLongFn("BITWISE_XOR", "bitwiseXor"))
.add(
OperatorConversions.druidUnaryLongFn(
"BITWISE_CONVERT_DOUBLE_TO_LONG_BITS",
"bitwiseConvertDoubleToLongBits"
)
)
.add(
OperatorConversions.druidUnaryDoubleFn(
"BITWISE_CONVERT_LONG_BITS_TO_DOUBLE",
"bitwiseConvertLongBitsToDouble"
)
)
.build();
private static final List<SqlOperatorConversion> STANDARD_OPERATOR_CONVERSIONS =
ImmutableList.<SqlOperatorConversion>builder()
.add(new DirectOperatorConversion(SqlStdOperatorTable.ABS, "abs"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.CASE, "case_searched"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.CHAR_LENGTH, "strlen"))
.add(CHARACTER_LENGTH_CONVERSION)
.add(new AliasedOperatorConversion(CHARACTER_LENGTH_CONVERSION, "LENGTH"))
.add(new AliasedOperatorConversion(CHARACTER_LENGTH_CONVERSION, "STRLEN"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.CONCAT, "concat"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.EXP, "exp"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.DIVIDE_INTEGER, "div"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.LN, "log"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.LOWER, "lower"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.LOG10, "log10"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.POWER, "pow"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.REPLACE, "replace"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.SQRT, "sqrt"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.UPPER, "upper"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.PI, "pi"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.SIN, "sin"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.COS, "cos"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.TAN, "tan"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.COT, "cot"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.ASIN, "asin"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.ACOS, "acos"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.ATAN, "atan"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.ATAN2, "atan2"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.RADIANS, "toRadians"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.DEGREES, "toDegrees"))
.add(new UnaryPrefixOperatorConversion(SqlStdOperatorTable.NOT, "!"))
.add(new UnaryPrefixOperatorConversion(SqlStdOperatorTable.UNARY_MINUS, "-"))
.add(new UnaryFunctionOperatorConversion(SqlStdOperatorTable.IS_NULL, "isnull"))
.add(new UnaryFunctionOperatorConversion(SqlStdOperatorTable.IS_NOT_NULL, "notnull"))
.add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_FALSE, "<= 0")) // Matches Evals.asBoolean
.add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_NOT_TRUE, "<= 0")) // Matches Evals.asBoolean
.add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_TRUE, "> 0")) // Matches Evals.asBoolean
.add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_NOT_FALSE, "> 0")) // Matches Evals.asBoolean
.add(new BinaryOperatorConversion(SqlStdOperatorTable.MULTIPLY, "*"))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.MOD, "%"))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.DIVIDE, "/"))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.PLUS, "+"))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.MINUS, "-"))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.EQUALS, "=="))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.NOT_EQUALS, "!="))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.GREATER_THAN, ">"))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, ">="))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.LESS_THAN, "<"))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, "<="))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.AND, "&&"))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.OR, "||"))
.add(new RoundOperatorConversion())
.addAll(TIME_OPERATOR_CONVERSIONS)
.addAll(STRING_OPERATOR_CONVERSIONS)
.addAll(VALUE_COERCION_OPERATOR_CONVERSIONS)
.addAll(ARRAY_OPERATOR_CONVERSIONS)
.addAll(MULTIVALUE_STRING_OPERATOR_CONVERSIONS)
.addAll(REDUCTION_OPERATOR_CONVERSIONS)
.addAll(IPV4ADDRESS_OPERATOR_CONVERSIONS)
.addAll(BITWISE_OPERATOR_CONVERSIONS)
.build();
.add(new DirectOperatorConversion(SqlStdOperatorTable.ABS, "abs"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.CASE, "case_searched"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.CHAR_LENGTH, "strlen"))
.add(CHARACTER_LENGTH_CONVERSION)
.add(new AliasedOperatorConversion(CHARACTER_LENGTH_CONVERSION, "LENGTH"))
.add(new AliasedOperatorConversion(CHARACTER_LENGTH_CONVERSION, "STRLEN"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.CONCAT, "concat"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.EXP, "exp"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.DIVIDE_INTEGER, "div"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.LN, "log"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.LOWER, "lower"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.LOG10, "log10"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.POWER, "pow"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.REPLACE, "replace"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.SQRT, "sqrt"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.UPPER, "upper"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.PI, "pi"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.SIN, "sin"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.COS, "cos"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.TAN, "tan"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.COT, "cot"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.ASIN, "asin"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.ACOS, "acos"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.ATAN, "atan"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.ATAN2, "atan2"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.RADIANS, "toRadians"))
.add(new DirectOperatorConversion(SqlStdOperatorTable.DEGREES, "toDegrees"))
.add(new UnaryPrefixOperatorConversion(SqlStdOperatorTable.NOT, "!"))
.add(new UnaryPrefixOperatorConversion(SqlStdOperatorTable.UNARY_MINUS, "-"))
.add(new UnaryFunctionOperatorConversion(SqlStdOperatorTable.IS_NULL, "isnull"))
.add(new UnaryFunctionOperatorConversion(SqlStdOperatorTable.IS_NOT_NULL, "notnull"))
.add(new UnarySuffixOperatorConversion(
SqlStdOperatorTable.IS_FALSE,
"<= 0"
)) // Matches Evals.asBoolean
.add(new UnarySuffixOperatorConversion(
SqlStdOperatorTable.IS_NOT_TRUE,
"<= 0"
)) // Matches Evals.asBoolean
.add(new UnarySuffixOperatorConversion(
SqlStdOperatorTable.IS_TRUE,
"> 0"
)) // Matches Evals.asBoolean
.add(new UnarySuffixOperatorConversion(
SqlStdOperatorTable.IS_NOT_FALSE,
"> 0"
)) // Matches Evals.asBoolean
.add(new BinaryOperatorConversion(SqlStdOperatorTable.MULTIPLY, "*"))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.MOD, "%"))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.DIVIDE, "/"))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.PLUS, "+"))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.MINUS, "-"))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.EQUALS, "=="))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.NOT_EQUALS, "!="))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.GREATER_THAN, ">"))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, ">="))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.LESS_THAN, "<"))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, "<="))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.AND, "&&"))
.add(new BinaryOperatorConversion(SqlStdOperatorTable.OR, "||"))
.add(new RoundOperatorConversion())
.addAll(TIME_OPERATOR_CONVERSIONS)
.addAll(STRING_OPERATOR_CONVERSIONS)
.addAll(VALUE_COERCION_OPERATOR_CONVERSIONS)
.addAll(ARRAY_OPERATOR_CONVERSIONS)
.addAll(MULTIVALUE_STRING_OPERATOR_CONVERSIONS)
.addAll(REDUCTION_OPERATOR_CONVERSIONS)
.addAll(IPV4ADDRESS_OPERATOR_CONVERSIONS)
.addAll(BITWISE_OPERATOR_CONVERSIONS)
.build();
// Operators that have no conversion, but are handled in the convertlet table, so they still need to exist.
private static final Map<OperatorKey, SqlOperator> CONVERTLET_OPERATORS =

20
sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java
View File

@ -28,6 +28,7 @@ import org.apache.calcite.avatica.remote.TypedValue;
import org.apache.calcite.linq4j.QueryProvider;
import org.apache.calcite.schema.SchemaPlus;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.Numbers;
import org.apache.druid.math.expr.ExprMacroTable;
import org.apache.druid.server.security.Access;
import org.apache.druid.server.security.AuthenticationResult;
@ -54,6 +55,7 @@ public class PlannerContext
public static final String CTX_SQL_QUERY_ID = "sqlQueryId";
public static final String CTX_SQL_CURRENT_TIMESTAMP = "sqlCurrentTimestamp";
public static final String CTX_SQL_TIME_ZONE = "sqlTimeZone";
public static final String CTX_SQL_STRINGIFY_ARRAYS = "sqlStringifyArrays";
// This context parameter is an undocumented parameter, used internally, to allow the web console to
// apply a limit without having to rewrite the SQL query.
@ -68,6 +70,7 @@ public class PlannerContext
private final DateTime localNow;
private final Map<String, Object> queryContext;
private final String sqlQueryId;
private final boolean stringifyArrays;
private final List<String> nativeQueryIds = new CopyOnWriteArrayList<>();
// bindings for dynamic parameters to bind during planning
private List<TypedValue> parameters = Collections.emptyList();
@ -83,6 +86,7 @@ public class PlannerContext
final ExprMacroTable macroTable,
final PlannerConfig plannerConfig,
final DateTime localNow,
final boolean stringifyArrays,
final Map<String, Object> queryContext
)
{
@ -91,6 +95,7 @@ public class PlannerContext
this.plannerConfig = Preconditions.checkNotNull(plannerConfig, "plannerConfig");
this.queryContext = queryContext != null ? new HashMap<>(queryContext) : new HashMap<>();
this.localNow = Preconditions.checkNotNull(localNow, "localNow");
this.stringifyArrays = stringifyArrays;
String sqlQueryId = (String) this.queryContext.get(CTX_SQL_QUERY_ID);
// special handling for DruidViewMacro, normal client will allocate sqlid in SqlLifecyle
@ -109,8 +114,10 @@ public class PlannerContext
{
final DateTime utcNow;
final DateTimeZone timeZone;
final boolean stringifyArrays;
if (queryContext != null) {
final Object stringifyParam = queryContext.get(CTX_SQL_STRINGIFY_ARRAYS);
final Object tsParam = queryContext.get(CTX_SQL_CURRENT_TIMESTAMP);
final Object tzParam = queryContext.get(CTX_SQL_TIME_ZONE);
@ -125,9 +132,16 @@ public class PlannerContext
} else {
timeZone = plannerConfig.getSqlTimeZone();
}
if (stringifyParam != null) {
stringifyArrays = Numbers.parseBoolean(stringifyParam);
} else {
stringifyArrays = true;
}
} else {
utcNow = new DateTime(DateTimeZone.UTC);
timeZone = plannerConfig.getSqlTimeZone();
stringifyArrays = true;
}
return new PlannerContext(
@ -135,6 +149,7 @@ public class PlannerContext
macroTable,
plannerConfig.withOverrides(queryContext),
utcNow.withZone(timeZone),
stringifyArrays,
queryContext
);
}
@ -169,6 +184,11 @@ public class PlannerContext
return queryContext;
}
public boolean isStringifyArrays()
{
return stringifyArrays;
}
public List<TypedValue> getParameters()
{
return parameters;

45
sql/src/main/java/org/apache/druid/sql/calcite/rel/QueryMaker.java
View File

@ -53,6 +53,8 @@ import org.joda.time.DateTime;
import org.joda.time.Interval;
import java.io.IOException;
import java.sql.Array;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.UUID;
@ -211,9 +213,11 @@ public class QueryMaker
} else if (sqlType == SqlTypeName.DATE) {
return ColumnMetaData.Rep.of(Integer.class);
} else if (sqlType == SqlTypeName.INTEGER) {
return ColumnMetaData.Rep.of(Integer.class);
// use Number.class for exact numeric types since JSON transport might switch longs to integers
return ColumnMetaData.Rep.of(Number.class);
} else if (sqlType == SqlTypeName.BIGINT) {
return ColumnMetaData.Rep.of(Long.class);
// use Number.class for exact numeric types since JSON transport might switch longs to integers
return ColumnMetaData.Rep.of(Number.class);
} else if (sqlType == SqlTypeName.FLOAT) {
return ColumnMetaData.Rep.of(Float.class);
} else if (sqlType == SqlTypeName.DOUBLE || sqlType == SqlTypeName.DECIMAL) {
@ -222,6 +226,8 @@ public class QueryMaker
return ColumnMetaData.Rep.of(Boolean.class);
} else if (sqlType == SqlTypeName.OTHER) {
return ColumnMetaData.Rep.of(Object.class);
} else if (sqlType == SqlTypeName.ARRAY) {
return ColumnMetaData.Rep.of(Array.class);
} else {
throw new ISE("No rep for SQL type[%s]", sqlType);
}
@ -309,16 +315,33 @@ public class QueryMaker
coercedValue = value.getClass().getName();
}
} else if (sqlType == SqlTypeName.ARRAY) {
if (value instanceof String) {
coercedValue = NullHandling.nullToEmptyIfNeeded((String) value);
} else if (value instanceof NlsString) {
coercedValue = ((NlsString) value).getValue();
} else {
try {
coercedValue = jsonMapper.writeValueAsString(value);
if (plannerContext.isStringifyArrays()) {
if (value instanceof String) {
coercedValue = NullHandling.nullToEmptyIfNeeded((String) value);
} else if (value instanceof NlsString) {
coercedValue = ((NlsString) value).getValue();
} else {
try {
coercedValue = jsonMapper.writeValueAsString(value);
}
catch (IOException e) {
throw new RuntimeException(e);
}
}
catch (IOException e) {
throw new RuntimeException(e);
} else {
// the protobuf jdbc handler prefers lists (it actually can't handle java arrays as sql arrays, only java lists)
// the json handler could handle this just fine, but it handles lists as sql arrays as well so just convert
// here if needed
if (value instanceof List) {
coercedValue = value;
} else if (value instanceof String[]) {
coercedValue = Arrays.asList((String[]) value);
} else if (value instanceof Long[]) {
coercedValue = Arrays.asList((Long[]) value);
} else if (value instanceof Double[]) {
coercedValue = Arrays.asList((Double[]) value);
} else {
throw new ISE("Cannot coerce[%s] to %s", value.getClass().getName(), sqlType);
}
}
} else {

37
sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java
View File

@ -81,6 +81,7 @@ import org.junit.rules.TemporaryFolder;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.sql.Array;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.Date;
@ -337,7 +338,7 @@ public abstract class DruidAvaticaHandlerTest extends CalciteTestBase
ImmutableList.of(
ImmutableMap.of(
"PLAN",
StringUtils.format("DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"%s\",\"sqlTimeZone\":\"America/Los_Angeles\"}}], signature=[{a0:LONG}])\n",
StringUtils.format("DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"%s\",\"sqlStringifyArrays\":false,\"sqlTimeZone\":\"America/Los_Angeles\"}}], signature=[{a0:LONG}])\n",
DUMMY_SQL_QUERY_ID
),
"RESOURCES",
@ -1316,6 +1317,33 @@ public abstract class DruidAvaticaHandlerTest extends CalciteTestBase
);
}
@Test
public void testArrayStuffs() throws Exception
{
PreparedStatement statement = client.prepareStatement(
"SELECT ARRAY_AGG(dim2) AS arr1, ARRAY_AGG(l1) AS arr2, ARRAY_AGG(d1) AS arr3, ARRAY_AGG(f1) AS arr4 FROM druid.numfoo"
);
final ResultSet resultSet = statement.executeQuery();
final List<Map<String, Object>> rows = getRows(resultSet);
Assert.assertEquals(1, rows.size());
Assert.assertTrue(rows.get(0).containsKey("arr1"));
Assert.assertTrue(rows.get(0).containsKey("arr2"));
Assert.assertTrue(rows.get(0).containsKey("arr3"));
Assert.assertTrue(rows.get(0).containsKey("arr4"));
if (NullHandling.sqlCompatible()) {
Assert.assertArrayEquals(new Object[]{"a", null, "", "a", "abc", null}, (Object[]) rows.get(0).get("arr1"));
Assert.assertArrayEquals(new Object[]{7L, 325323L, 0L, null, null, null}, (Object[]) rows.get(0).get("arr2"));
Assert.assertArrayEquals(new Object[]{1.0, 1.7, 0.0, null, null, null}, (Object[]) rows.get(0).get("arr3"));
Assert.assertArrayEquals(new Object[]{1.0f, 0.1f, 0.0f, null, null, null}, (Object[]) rows.get(0).get("arr4"));
} else {
Assert.assertArrayEquals(new Object[]{"a", null, null, "a", "abc", null}, (Object[]) rows.get(0).get("arr1"));
Assert.assertArrayEquals(new Object[]{7L, 325323L, 0L, 0L, 0L, 0L}, (Object[]) rows.get(0).get("arr2"));
Assert.assertArrayEquals(new Object[]{1.0, 1.7, 0.0, 0.0, 0.0, 0.0}, (Object[]) rows.get(0).get("arr3"));
Assert.assertArrayEquals(new Object[]{1.0f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f}, (Object[]) rows.get(0).get("arr4"));
}
}
protected abstract String getJdbcConnectionString(int port);
protected abstract AbstractAvaticaHandler getAvaticaHandler(DruidMeta druidMeta);
@ -1335,7 +1363,12 @@ public abstract class DruidAvaticaHandlerTest extends CalciteTestBase
final Map<String, Object> row = new HashMap<>();
for (int i = 0; i < metaData.getColumnCount(); i++) {
if (returnKeys == null || returnKeys.contains(metaData.getColumnLabel(i + 1))) {
row.put(metaData.getColumnLabel(i + 1), resultSet.getObject(i + 1));
Object result = resultSet.getObject(i + 1);
if (result instanceof Array) {
row.put(metaData.getColumnLabel(i + 1), ((Array) result).getArray());
} else {
row.put(metaData.getColumnLabel(i + 1), result);
}
}
}
rows.add(row);

2
sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java
View File

@ -124,7 +124,7 @@ public class DruidStatementTest extends CalciteTestBase
Assert.assertEquals(
Lists.newArrayList(
Lists.newArrayList("__time", "TIMESTAMP", "java.lang.Long"),
Lists.newArrayList("cnt", "BIGINT", "java.lang.Long"),
Lists.newArrayList("cnt", "BIGINT", "java.lang.Number"),
Lists.newArrayList("dim1", "VARCHAR", "java.lang.String"),
Lists.newArrayList("dim2", "VARCHAR", "java.lang.String"),
Lists.newArrayList("dim3", "VARCHAR", "java.lang.String"),

6
sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
View File

@ -112,6 +112,8 @@ public class BaseCalciteQueryTest extends CalciteTestBase
public static Long NULL_LONG;
public static final String HLLC_STRING = VersionOneHyperLogLogCollector.class.getName();
final boolean useDefault = NullHandling.replaceWithDefault();
@BeforeClass
public static void setupNullValues()
{
@ -175,6 +177,10 @@ public class BaseCalciteQueryTest extends CalciteTestBase
.put(QueryContexts.MAX_SCATTER_GATHER_BYTES_KEY, Long.MAX_VALUE);
public static final Map<String, Object> QUERY_CONTEXT_DEFAULT = DEFAULT_QUERY_CONTEXT_BUILDER.build();
public static final Map<String, Object> QUERY_CONTEXT_NO_STRINGIFY_ARRAY =
DEFAULT_QUERY_CONTEXT_BUILDER.put(PlannerContext.CTX_SQL_STRINGIFY_ARRAYS, false)
.build();
public static final Map<String, Object> QUERY_CONTEXT_DONT_SKIP_EMPTY_BUCKETS = ImmutableMap.of(
PlannerContext.CTX_SQL_QUERY_ID, DUMMY_SQL_ID,
PlannerContext.CTX_SQL_CURRENT_TIMESTAMP, "2000-01-01T00:00:00Z",

1763
sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java Normal file
View File

File diff suppressed because it is too large Load Diff

936
sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java Normal file
View File

@ -0,0 +1,936 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.sql.calcite;
import com.google.common.collect.ImmutableList;
import junitparams.JUnitParamsRunner;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.query.Druids;
import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
import org.apache.druid.query.filter.AndDimFilter;
import org.apache.druid.query.filter.InDimFilter;
import org.apache.druid.query.filter.SelectorDimFilter;
import org.apache.druid.query.groupby.GroupByQuery;
import org.apache.druid.query.groupby.orderby.DefaultLimitSpec;
import org.apache.druid.query.groupby.orderby.OrderByColumnSpec;
import org.apache.druid.query.ordering.StringComparators;
import org.apache.druid.query.scan.ScanQuery;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.sql.calcite.filtration.Filtration;
import org.apache.druid.sql.calcite.util.CalciteTests;
import org.junit.Test;
import org.junit.runner.RunWith;
import java.util.List;
@RunWith(JUnitParamsRunner.class)
public class CalciteMultiValueStringQueryTest extends BaseCalciteQueryTest
{
// various queries on multi-valued string dimensions using them like strings
@Test
public void testMultiValueStringWorksLikeStringGroupBy() throws Exception
{
// Cannot vectorize due to usage of expressions.
cannotVectorize();
List<Object[]> expected;
if (NullHandling.replaceWithDefault()) {
expected = ImmutableList.of(
new Object[]{"bfoo", 2L},
new Object[]{"foo", 2L},
new Object[]{"", 1L},
new Object[]{"afoo", 1L},
new Object[]{"cfoo", 1L},
new Object[]{"dfoo", 1L}
);
} else {
expected = ImmutableList.of(
new Object[]{null, 2L},
new Object[]{"bfoo", 2L},
new Object[]{"afoo", 1L},
new Object[]{"cfoo", 1L},
new Object[]{"dfoo", 1L},
new Object[]{"foo", 1L}
);
}
testQuery(
"SELECT concat(dim3, 'foo'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
GroupByQuery.builder()
.setDataSource(CalciteTests.DATASOURCE3)
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
.setVirtualColumns(expressionVirtualColumn("v0", "concat(\"dim3\",'foo')", ValueType.STRING))
.setDimensions(
dimensions(
new DefaultDimensionSpec("v0", "_d0", ValueType.STRING)
)
)
.setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt")))
.setLimitSpec(new DefaultLimitSpec(
ImmutableList.of(new OrderByColumnSpec(
"a0",
OrderByColumnSpec.Direction.DESCENDING,
StringComparators.NUMERIC
)),
Integer.MAX_VALUE
))
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
),
expected
);
}
@Test
public void testMultiValueStringWorksLikeStringGroupByWithFilter() throws Exception
{
// Cannot vectorize due to usage of expressions.
cannotVectorize();
testQuery(
"SELECT concat(dim3, 'foo'), SUM(cnt) FROM druid.numfoo where concat(dim3, 'foo') = 'bfoo' GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
GroupByQuery.builder()
.setDataSource(CalciteTests.DATASOURCE3)
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
.setVirtualColumns(expressionVirtualColumn("v0", "concat(\"dim3\",'foo')", ValueType.STRING))
.setDimensions(
dimensions(
new DefaultDimensionSpec("v0", "_d0", ValueType.STRING)
)
)
.setDimFilter(selector("v0", "bfoo", null))
.setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt")))
.setLimitSpec(new DefaultLimitSpec(
ImmutableList.of(new OrderByColumnSpec(
"a0",
OrderByColumnSpec.Direction.DESCENDING,
StringComparators.NUMERIC
)),
Integer.MAX_VALUE
))
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(
new Object[]{"bfoo", 2L},
new Object[]{"afoo", 1L},
new Object[]{"cfoo", 1L}
)
);
}
@Test
public void testMultiValueStringWorksLikeStringScan() throws Exception
{
final String nullVal = NullHandling.replaceWithDefault() ? "[\"foo\"]" : "[null]";
testQuery(
"SELECT concat(dim3, 'foo') FROM druid.numfoo",
ImmutableList.of(
new Druids.ScanQueryBuilder()
.dataSource(CalciteTests.DATASOURCE3)
.intervals(querySegmentSpec(Filtration.eternity()))
.virtualColumns(expressionVirtualColumn("v0", "concat(\"dim3\",'foo')", ValueType.STRING))
.columns(ImmutableList.of("v0"))
.context(QUERY_CONTEXT_DEFAULT)
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.legacy(false)
.build()
),
ImmutableList.of(
new Object[]{"[\"afoo\",\"bfoo\"]"},
new Object[]{"[\"bfoo\",\"cfoo\"]"},
new Object[]{"[\"dfoo\"]"},
new Object[]{"[\"foo\"]"},
new Object[]{nullVal},
new Object[]{nullVal}
)
);
}
@Test
public void testMultiValueStringWorksLikeStringSelfConcatScan() throws Exception
{
final String nullVal = NullHandling.replaceWithDefault() ? "[\"-lol-\"]" : "[null]";
testQuery(
"SELECT concat(dim3, '-lol-', dim3) FROM druid.numfoo",
ImmutableList.of(
new Druids.ScanQueryBuilder()
.dataSource(CalciteTests.DATASOURCE3)
.intervals(querySegmentSpec(Filtration.eternity()))
.virtualColumns(expressionVirtualColumn("v0", "concat(\"dim3\",'-lol-',\"dim3\")", ValueType.STRING))
.columns(ImmutableList.of("v0"))
.context(QUERY_CONTEXT_DEFAULT)
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.legacy(false)
.build()
),
ImmutableList.of(
new Object[]{"[\"a-lol-a\",\"b-lol-b\"]"},
new Object[]{"[\"b-lol-b\",\"c-lol-c\"]"},
new Object[]{"[\"d-lol-d\"]"},
new Object[]{"[\"-lol-\"]"},
new Object[]{nullVal},
new Object[]{nullVal}
)
);
}
@Test
public void testMultiValueStringWorksLikeStringScanWithFilter() throws Exception
{
testQuery(
"SELECT concat(dim3, 'foo') FROM druid.numfoo where concat(dim3, 'foo') = 'bfoo'",
ImmutableList.of(
new Druids.ScanQueryBuilder()
.dataSource(CalciteTests.DATASOURCE3)
.intervals(querySegmentSpec(Filtration.eternity()))
.virtualColumns(expressionVirtualColumn("v0", "concat(\"dim3\",'foo')", ValueType.STRING))
.filters(selector("v0", "bfoo", null))
.columns(ImmutableList.of("v0"))
.context(QUERY_CONTEXT_DEFAULT)
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.legacy(false)
.build()
),
ImmutableList.of(
new Object[]{"[\"afoo\",\"bfoo\"]"},
new Object[]{"[\"bfoo\",\"cfoo\"]"}
)
);
}
// these are a copy of the ARRAY functions tests in CalciteArraysQueryTest
@Test
public void testMultiValueStringOverlapFilter() throws Exception
{
testQuery(
"SELECT dim3 FROM druid.numfoo WHERE MV_OVERLAP(dim3, ARRAY['a','b']) LIMIT 5",
ImmutableList.of(
newScanQueryBuilder()
.dataSource(CalciteTests.DATASOURCE3)
.intervals(querySegmentSpec(Filtration.eternity()))
.filters(new InDimFilter("dim3", ImmutableList.of("a", "b"), null))
.columns("dim3")
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.limit(5)
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(
new Object[]{"[\"a\",\"b\"]"},
new Object[]{"[\"b\",\"c\"]"}
)
);
}
@Test
public void testMultiValueStringOverlapFilterNonLiteral() throws Exception
{
testQuery(
"SELECT dim3 FROM druid.numfoo WHERE MV_OVERLAP(dim3, ARRAY[dim2]) LIMIT 5",
ImmutableList.of(
newScanQueryBuilder()
.dataSource(CalciteTests.DATASOURCE3)
.intervals(querySegmentSpec(Filtration.eternity()))
.filters(expressionFilter("array_overlap(\"dim3\",array(\"dim2\"))"))
.columns("dim3")
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.limit(5)
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(
new Object[]{"[\"a\",\"b\"]"},
new Object[]{useDefault ? "" : null}
)
);
}
@Test
public void testMultiValueStringContainsFilter() throws Exception
{
testQuery(
"SELECT dim3 FROM druid.numfoo WHERE MV_CONTAINS(dim3, ARRAY['a','b']) LIMIT 5",
ImmutableList.of(
newScanQueryBuilder()
.dataSource(CalciteTests.DATASOURCE3)
.intervals(querySegmentSpec(Filtration.eternity()))
.filters(
new AndDimFilter(
new SelectorDimFilter("dim3", "a", null),
new SelectorDimFilter("dim3", "b", null)
)
)
.columns("dim3")
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.limit(5)
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(
new Object[]{"[\"a\",\"b\"]"}
)
);
}
@Test
public void testMultiValueStringContainsArrayOfOneElement() throws Exception
{
testQuery(
"SELECT dim3 FROM druid.numfoo WHERE MV_CONTAINS(dim3, ARRAY['a']) LIMIT 5",
ImmutableList.of(
newScanQueryBuilder()
.dataSource(CalciteTests.DATASOURCE3)
.intervals(querySegmentSpec(Filtration.eternity()))
.filters(new SelectorDimFilter("dim3", "a", null))
.columns("dim3")
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.limit(5)
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(
new Object[]{"[\"a\",\"b\"]"}
)
);
}
@Test
public void testMultiValueStringContainsArrayOfNonLiteral() throws Exception
{
testQuery(
"SELECT dim3 FROM druid.numfoo WHERE MV_CONTAINS(dim3, ARRAY[dim2]) LIMIT 5",
ImmutableList.of(
newScanQueryBuilder()
.dataSource(CalciteTests.DATASOURCE3)
.intervals(querySegmentSpec(Filtration.eternity()))
.filters(expressionFilter("array_contains(\"dim3\",array(\"dim2\"))"))
.columns("dim3")
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.limit(5)
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(
new Object[]{"[\"a\",\"b\"]"},
new Object[]{useDefault ? "" : null}
)
);
}
@Test
public void testMultiValueStringSlice() throws Exception
{
testQuery(
"SELECT MV_SLICE(dim3, 1) FROM druid.numfoo",
ImmutableList.of(
new Druids.ScanQueryBuilder()
.dataSource(CalciteTests.DATASOURCE3)
.intervals(querySegmentSpec(Filtration.eternity()))
.virtualColumns(expressionVirtualColumn("v0", "array_slice(\"dim3\",1)", ValueType.STRING))
.columns(ImmutableList.of("v0"))
.context(QUERY_CONTEXT_DEFAULT)
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.legacy(false)
.build()
),
ImmutableList.of(
new Object[]{"[\"b\"]"},
new Object[]{"[\"c\"]"},
new Object[]{"[]"},
new Object[]{"[]"},
new Object[]{"[]"},
new Object[]{"[]"}
)
);
}
@Test
public void testMultiValueStringLength() throws Exception
{
// Cannot vectorize due to usage of expressions.
cannotVectorize();
testQuery(
"SELECT dim1, MV_LENGTH(dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1, 2 ORDER BY 2 DESC",
ImmutableList.of(
GroupByQuery.builder()
.setDataSource(CalciteTests.DATASOURCE3)
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
.setVirtualColumns(expressionVirtualColumn("v0", "array_length(\"dim3\")", ValueType.LONG))
.setDimensions(
dimensions(
new DefaultDimensionSpec("dim1", "_d0", ValueType.STRING),
new DefaultDimensionSpec("v0", "_d1", ValueType.LONG)
)
)
.setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt")))
.setLimitSpec(new DefaultLimitSpec(
ImmutableList.of(new OrderByColumnSpec(
"_d1",
OrderByColumnSpec.Direction.DESCENDING,
StringComparators.NUMERIC
)),
Integer.MAX_VALUE
))
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(
new Object[]{"", 2, 1L},
new Object[]{"10.1", 2, 1L},
new Object[]{"1", 1, 1L},
new Object[]{"2", 1, 1L},
new Object[]{"abc", 1, 1L},
new Object[]{"def", 1, 1L}
)
);
}
@Test
public void testMultiValueStringAppend() throws Exception
{
// Cannot vectorize due to usage of expressions.
cannotVectorize();
ImmutableList<Object[]> results;
if (useDefault) {
results = ImmutableList.of(
new Object[]{"foo", 6L},
new Object[]{"", 3L},
new Object[]{"b", 2L},
new Object[]{"a", 1L},
new Object[]{"c", 1L},
new Object[]{"d", 1L}
);
} else {
results = ImmutableList.of(
new Object[]{"foo", 6L},
new Object[]{null, 2L},
new Object[]{"b", 2L},
new Object[]{"", 1L},
new Object[]{"a", 1L},
new Object[]{"c", 1L},
new Object[]{"d", 1L}
);
}
testQuery(
"SELECT MV_APPEND(dim3, 'foo'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
GroupByQuery.builder()
.setDataSource(CalciteTests.DATASOURCE3)
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
.setVirtualColumns(expressionVirtualColumn(
"v0",
"array_append(\"dim3\",'foo')",
ValueType.STRING
))
.setDimensions(
dimensions(
new DefaultDimensionSpec("v0", "_d0", ValueType.STRING)
)
)
.setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt")))
.setLimitSpec(new DefaultLimitSpec(
ImmutableList.of(new OrderByColumnSpec(
"a0",
OrderByColumnSpec.Direction.DESCENDING,
StringComparators.NUMERIC
)),
Integer.MAX_VALUE
))
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
),
results
);
}
@Test
public void testMultiValueStringPrepend() throws Exception
{
// Cannot vectorize due to usage of expressions.
cannotVectorize();
ImmutableList<Object[]> results;
if (useDefault) {
results = ImmutableList.of(
new Object[]{"foo", 6L},
new Object[]{"", 3L},
new Object[]{"b", 2L},
new Object[]{"a", 1L},
new Object[]{"c", 1L},
new Object[]{"d", 1L}
);
} else {
results = ImmutableList.of(
new Object[]{"foo", 6L},
new Object[]{null, 2L},
new Object[]{"b", 2L},
new Object[]{"", 1L},
new Object[]{"a", 1L},
new Object[]{"c", 1L},
new Object[]{"d", 1L}
);
}
testQuery(
"SELECT MV_PREPEND('foo', dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
GroupByQuery.builder()
.setDataSource(CalciteTests.DATASOURCE3)
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
.setVirtualColumns(expressionVirtualColumn(
"v0",
"array_prepend('foo',\"dim3\")",
ValueType.STRING
))
.setDimensions(
dimensions(
new DefaultDimensionSpec("v0", "_d0", ValueType.STRING)
)
)
.setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt")))
.setLimitSpec(new DefaultLimitSpec(
ImmutableList.of(new OrderByColumnSpec(
"a0",
OrderByColumnSpec.Direction.DESCENDING,
StringComparators.NUMERIC
)),
Integer.MAX_VALUE
))
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
),
results
);
}
@Test
public void testMultiValueStringPrependAppend() throws Exception
{
// Cannot vectorize due to usage of expressions.
cannotVectorize();
ImmutableList<Object[]> results;
if (useDefault) {
results = ImmutableList.of(
new Object[]{"foo,null", "null,foo", 3L},
new Object[]{"foo,a,b", "a,b,foo", 1L},
new Object[]{"foo,b,c", "b,c,foo", 1L},
new Object[]{"foo,d", "d,foo", 1L}
);
} else {
results = ImmutableList.of(
new Object[]{"foo,null", "null,foo", 2L},
new Object[]{"foo,", ",foo", 1L},
new Object[]{"foo,a,b", "a,b,foo", 1L},
new Object[]{"foo,b,c", "b,c,foo", 1L},
new Object[]{"foo,d", "d,foo", 1L}
);
}
testQuery(
"SELECT MV_TO_STRING(MV_PREPEND('foo', dim3), ','), MV_TO_STRING(MV_APPEND(dim3, 'foo'), ','), SUM(cnt) FROM druid.numfoo GROUP BY 1,2 ORDER BY 3 DESC",
ImmutableList.of(
GroupByQuery.builder()
.setDataSource(CalciteTests.DATASOURCE3)
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
.setVirtualColumns(
expressionVirtualColumn(
"v0",
"array_to_string(array_prepend('foo',\"dim3\"),',')",
ValueType.STRING
),
expressionVirtualColumn(
"v1",
"array_to_string(array_append(\"dim3\",'foo'),',')",
ValueType.STRING
)
)
.setDimensions(
dimensions(
new DefaultDimensionSpec("v0", "_d0", ValueType.STRING),
new DefaultDimensionSpec("v1", "_d1", ValueType.STRING)
)
)
.setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt")))
.setLimitSpec(new DefaultLimitSpec(
ImmutableList.of(new OrderByColumnSpec(
"a0",
OrderByColumnSpec.Direction.DESCENDING,
StringComparators.NUMERIC
)),
Integer.MAX_VALUE
))
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
),
results
);
}
@Test
public void testMultiValueStringConcat() throws Exception
{
// Cannot vectorize due to usage of expressions.
cannotVectorize();
ImmutableList<Object[]> results;
if (useDefault) {
results = ImmutableList.of(
new Object[]{"", 6L},
new Object[]{"b", 4L},
new Object[]{"a", 2L},
new Object[]{"c", 2L},
new Object[]{"d", 2L}
);
} else {
results = ImmutableList.of(
new Object[]{null, 4L},
new Object[]{"b", 4L},
new Object[]{"", 2L},
new Object[]{"a", 2L},
new Object[]{"c", 2L},
new Object[]{"d", 2L}
);
}
testQuery(
"SELECT MV_CONCAT(dim3, dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
GroupByQuery.builder()
.setDataSource(CalciteTests.DATASOURCE3)
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
.setVirtualColumns(expressionVirtualColumn(
"v0",
"array_concat(\"dim3\",\"dim3\")",
ValueType.STRING
))
.setDimensions(
dimensions(
new DefaultDimensionSpec("v0", "_d0", ValueType.STRING)
)
)
.setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt")))
.setLimitSpec(new DefaultLimitSpec(
ImmutableList.of(new OrderByColumnSpec(
"a0",
OrderByColumnSpec.Direction.DESCENDING,
StringComparators.NUMERIC
)),
Integer.MAX_VALUE
))
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
),
results
);
}
@Test
public void testMultiValueStringOffset() throws Exception
{
// Cannot vectorize due to usage of expressions.
cannotVectorize();
testQuery(
"SELECT MV_OFFSET(dim3, 1), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
GroupByQuery.builder()
.setDataSource(CalciteTests.DATASOURCE3)
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
.setVirtualColumns(expressionVirtualColumn("v0", "array_offset(\"dim3\",1)", ValueType.STRING))
.setDimensions(
dimensions(
new DefaultDimensionSpec("v0", "_d0", ValueType.STRING)
)
)
.setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt")))
.setLimitSpec(new DefaultLimitSpec(
ImmutableList.of(new OrderByColumnSpec(
"a0",
OrderByColumnSpec.Direction.DESCENDING,
StringComparators.NUMERIC
)),
Integer.MAX_VALUE
))
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(
new Object[]{NullHandling.defaultStringValue(), 4L},
new Object[]{"b", 1L},
new Object[]{"c", 1L}
)
);
}
@Test
public void testMultiValueStringOrdinal() throws Exception
{
// Cannot vectorize due to usage of expressions.
cannotVectorize();
testQuery(
"SELECT MV_ORDINAL(dim3, 2), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
GroupByQuery.builder()
.setDataSource(CalciteTests.DATASOURCE3)
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
.setVirtualColumns(expressionVirtualColumn("v0", "array_ordinal(\"dim3\",2)", ValueType.STRING))
.setDimensions(
dimensions(
new DefaultDimensionSpec("v0", "_d0", ValueType.STRING)
)
)
.setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt")))
.setLimitSpec(new DefaultLimitSpec(
ImmutableList.of(new OrderByColumnSpec(
"a0",
OrderByColumnSpec.Direction.DESCENDING,
StringComparators.NUMERIC
)),
Integer.MAX_VALUE
))
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(
new Object[]{NullHandling.defaultStringValue(), 4L},
new Object[]{"b", 1L},
new Object[]{"c", 1L}
)
);
}
@Test
public void testMultiValueStringOffsetOf() throws Exception
{
// Cannot vectorize due to usage of expressions.
cannotVectorize();
testQuery(
"SELECT MV_OFFSET_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
GroupByQuery.builder()
.setDataSource(CalciteTests.DATASOURCE3)
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
.setVirtualColumns(expressionVirtualColumn(
"v0",
"array_offset_of(\"dim3\",'b')",
ValueType.LONG
))
.setDimensions(
dimensions(
new DefaultDimensionSpec("v0", "_d0", ValueType.LONG)
)
)
.setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt")))
.setLimitSpec(new DefaultLimitSpec(
ImmutableList.of(new OrderByColumnSpec(
"a0",
OrderByColumnSpec.Direction.DESCENDING,
StringComparators.NUMERIC
)),
Integer.MAX_VALUE
))
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(
new Object[]{useDefault ? -1 : null, 4L},
new Object[]{0, 1L},
new Object[]{1, 1L}
)
);
}
@Test
public void testMultiValueStringOrdinalOf() throws Exception
{
// Cannot vectorize due to usage of expressions.
cannotVectorize();
testQuery(
"SELECT MV_ORDINAL_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
GroupByQuery.builder()
.setDataSource(CalciteTests.DATASOURCE3)
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
.setVirtualColumns(expressionVirtualColumn(
"v0",
"array_ordinal_of(\"dim3\",'b')",
ValueType.LONG
))
.setDimensions(
dimensions(
new DefaultDimensionSpec("v0", "_d0", ValueType.LONG)
)
)
.setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt")))
.setLimitSpec(new DefaultLimitSpec(
ImmutableList.of(new OrderByColumnSpec(
"a0",
OrderByColumnSpec.Direction.DESCENDING,
StringComparators.NUMERIC
)),
Integer.MAX_VALUE
))
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(
new Object[]{useDefault ? -1 : null, 4L},
new Object[]{1, 1L},
new Object[]{2, 1L}
)
);
}
@Test
public void testMultiValueStringToString() throws Exception
{
// Cannot vectorize due to usage of expressions.
cannotVectorize();
ImmutableList<Object[]> results;
if (useDefault) {
results = ImmutableList.of(
new Object[]{"", 3L},
new Object[]{"a,b", 1L},
new Object[]{"b,c", 1L},
new Object[]{"d", 1L}
);
} else {
results = ImmutableList.of(
new Object[]{null, 2L},
new Object[]{"", 1L},
new Object[]{"a,b", 1L},
new Object[]{"b,c", 1L},
new Object[]{"d", 1L}
);
}
testQuery(
"SELECT MV_TO_STRING(dim3, ','), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
GroupByQuery.builder()
.setDataSource(CalciteTests.DATASOURCE3)
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
.setVirtualColumns(expressionVirtualColumn(
"v0",
"array_to_string(\"dim3\",',')",
ValueType.STRING
))
.setDimensions(
dimensions(
new DefaultDimensionSpec("v0", "_d0", ValueType.STRING)
)
)
.setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt")))
.setLimitSpec(new DefaultLimitSpec(
ImmutableList.of(new OrderByColumnSpec(
"a0",
OrderByColumnSpec.Direction.DESCENDING,
StringComparators.NUMERIC
)),
Integer.MAX_VALUE
))
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
),
results
);
}
@Test
public void testMultiValueStringToStringToMultiValueString() throws Exception
{
// Cannot vectorize due to usage of expressions.
cannotVectorize();
ImmutableList<Object[]> results;
if (useDefault) {
results = ImmutableList.of(
new Object[]{"d", 7L},
new Object[]{"", 3L},
new Object[]{"b", 2L},
new Object[]{"a", 1L},
new Object[]{"c", 1L}
);
} else {
results = ImmutableList.of(
new Object[]{"d", 5L},
new Object[]{null, 2L},
new Object[]{"b", 2L},
new Object[]{"", 1L},
new Object[]{"a", 1L},
new Object[]{"c", 1L}
);
}
testQuery(
"SELECT STRING_TO_MV(CONCAT(MV_TO_STRING(dim3, ','), ',d'), ','), SUM(cnt) FROM druid.numfoo WHERE MV_LENGTH(dim3) > 0 GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
GroupByQuery.builder()
.setDataSource(CalciteTests.DATASOURCE3)
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
.setVirtualColumns(
expressionVirtualColumn("v0", "array_length(\"dim3\")", ValueType.LONG),
expressionVirtualColumn(
"v1",
"string_to_array(concat(array_to_string(\"dim3\",','),',d'),',')",
ValueType.STRING
)
)
.setDimFilter(bound("v0", "0", null, true, false, null, StringComparators.NUMERIC))
.setDimensions(
dimensions(
new DefaultDimensionSpec("v1", "_d0", ValueType.STRING)
)
)
.setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt")))
.setLimitSpec(new DefaultLimitSpec(
ImmutableList.of(new OrderByColumnSpec(
"a0",
OrderByColumnSpec.Direction.DESCENDING,
StringComparators.NUMERIC
)),
Integer.MAX_VALUE
))
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
),
results
);
}
}

5
sql/src/test/java/org/apache/druid/sql/calcite/CalciteParameterQueryTest.java
View File

@ -20,6 +20,7 @@
package org.apache.druid.sql.calcite;
import com.google.common.collect.ImmutableList;
import junitparams.JUnitParamsRunner;
import org.apache.calcite.avatica.SqlType;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.DateTimes;
@ -41,6 +42,7 @@ import org.apache.druid.sql.calcite.filtration.Filtration;
import org.apache.druid.sql.calcite.util.CalciteTests;
import org.apache.druid.sql.http.SqlParameter;
import org.junit.Test;
import org.junit.runner.RunWith;
/**
* This class has copied a subset of the tests in {@link CalciteQueryTest} and replaced various parts of queries with
@ -48,10 +50,9 @@ import org.junit.Test;
* were merely chosen to produce a selection of parameter types and positions within query expressions and have been
* renamed to reflect this
*/
@RunWith(JUnitParamsRunner.class)
public class CalciteParameterQueryTest extends BaseCalciteQueryTest
{
private final boolean useDefault = NullHandling.replaceWithDefault();
@Test
public void testSelectConstantParamGetsConstant() throws Exception
{

1607
sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
View File

File diff suppressed because it is too large Load Diff