mirror of https://github.com/apache/druid.git
fix ColumnType to RelDataType conversion for nested arrays (#16138)
* fix ColumnType to RelDataType conversion for nested arrays * fix test
This commit is contained in:
Clint Wylie
GitHub
parent
c7823bca98
commit
5afd5c41a5
|
|
@ -96,8 +96,8 @@ public class RowSignatures
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns a Calcite RelDataType corresponding to a row signature. It will typecast __time column to TIMESTAMP
|
* Returns a Calcite {@link RelDataType} corresponding to a {@link RowSignature}. It will typecast __time column to
|
||||||
* irrespective of the type present in the row signature
|
* TIMESTAMP irrespective of the type present in the row signature
|
||||||
*/
|
*/
|
||||||
public static RelDataType toRelDataType(final RowSignature rowSignature, final RelDataTypeFactory typeFactory)
|
public static RelDataType toRelDataType(final RowSignature rowSignature, final RelDataTypeFactory typeFactory)
|
||||||
{
|
{
|
||||||
|
|
@ -105,8 +105,8 @@ public class RowSignatures
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns a Calcite RelDataType corresponding to a row signature.
|
* Returns a Calcite {@link RelDataType} corresponding to a {@link RowSignature}. For columns that are named
|
||||||
* For columns that are named "__time", it automatically casts it to TIMESTAMP if typecastTimeColumn is set to true
|
* "__time", it automatically casts it to TIMESTAMP if typecastTimeColumn is set to true
|
||||||
*/
|
*/
|
||||||
public static RelDataType toRelDataType(
|
public static RelDataType toRelDataType(
|
||||||
final RowSignature rowSignature,
|
final RowSignature rowSignature,
|
||||||
|
|
@ -126,44 +126,7 @@ public class RowSignatures
|
||||||
rowSignature.getColumnType(columnName)
|
rowSignature.getColumnType(columnName)
|
||||||
.orElseThrow(() -> new ISE("Encountered null type for column[%s]", columnName));
|
.orElseThrow(() -> new ISE("Encountered null type for column[%s]", columnName));
|
||||||
|
|
||||||
switch (columnType.getType()) {
|
type = columnTypeToRelDataType(typeFactory, columnType, nullNumeric);
|
||||||
case STRING:
|
|
||||||
// Note that there is no attempt here to handle multi-value in any special way. Maybe one day...
|
|
||||||
type = Calcites.createSqlTypeWithNullability(typeFactory, SqlTypeName.VARCHAR, true);
|
|
||||||
break;
|
|
||||||
case LONG:
|
|
||||||
type = Calcites.createSqlTypeWithNullability(typeFactory, SqlTypeName.BIGINT, nullNumeric);
|
|
||||||
break;
|
|
||||||
case FLOAT:
|
|
||||||
type = Calcites.createSqlTypeWithNullability(typeFactory, SqlTypeName.FLOAT, nullNumeric);
|
|
||||||
break;
|
|
||||||
case DOUBLE:
|
|
||||||
type = Calcites.createSqlTypeWithNullability(typeFactory, SqlTypeName.DOUBLE, nullNumeric);
|
|
||||||
break;
|
|
||||||
case ARRAY:
|
|
||||||
switch (columnType.getElementType().getType()) {
|
|
||||||
case STRING:
|
|
||||||
type = Calcites.createSqlArrayTypeWithNullability(typeFactory, SqlTypeName.VARCHAR, true);
|
|
||||||
break;
|
|
||||||
case LONG:
|
|
||||||
type = Calcites.createSqlArrayTypeWithNullability(typeFactory, SqlTypeName.BIGINT, nullNumeric);
|
|
||||||
break;
|
|
||||||
case DOUBLE:
|
|
||||||
type = Calcites.createSqlArrayTypeWithNullability(typeFactory, SqlTypeName.DOUBLE, nullNumeric);
|
|
||||||
break;
|
|
||||||
case FLOAT:
|
|
||||||
type = Calcites.createSqlArrayTypeWithNullability(typeFactory, SqlTypeName.FLOAT, nullNumeric);
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
throw new ISE("valueType[%s] not translatable", columnType);
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
case COMPLEX:
|
|
||||||
type = makeComplexType(typeFactory, columnType, true);
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
throw new ISE("valueType[%s] not translatable", columnType);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
builder.add(columnName, type);
|
builder.add(columnName, type);
|
||||||
|
|
@ -172,6 +135,49 @@ public class RowSignatures
|
||||||
return builder.build();
|
return builder.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a Calcite {@link RelDataType} corresponding to a {@link ColumnType}
|
||||||
|
*/
|
||||||
|
public static RelDataType columnTypeToRelDataType(
|
||||||
|
RelDataTypeFactory typeFactory,
|
||||||
|
ColumnType columnType,
|
||||||
|
boolean nullNumeric
|
||||||
|
)
|
||||||
|
{
|
||||||
|
final RelDataType type;
|
||||||
|
switch (columnType.getType()) {
|
||||||
|
case STRING:
|
||||||
|
// Note that there is no attempt here to handle multi-value in any special way. Maybe one day...
|
||||||
|
type = Calcites.createSqlTypeWithNullability(typeFactory, SqlTypeName.VARCHAR, true);
|
||||||
|
break;
|
||||||
|
case LONG:
|
||||||
|
type = Calcites.createSqlTypeWithNullability(typeFactory, SqlTypeName.BIGINT, nullNumeric);
|
||||||
|
break;
|
||||||
|
case FLOAT:
|
||||||
|
type = Calcites.createSqlTypeWithNullability(typeFactory, SqlTypeName.FLOAT, nullNumeric);
|
||||||
|
break;
|
||||||
|
case DOUBLE:
|
||||||
|
type = Calcites.createSqlTypeWithNullability(typeFactory, SqlTypeName.DOUBLE, nullNumeric);
|
||||||
|
break;
|
||||||
|
case ARRAY:
|
||||||
|
final RelDataType elementType = columnTypeToRelDataType(
|
||||||
|
typeFactory,
|
||||||
|
(ColumnType) columnType.getElementType(),
|
||||||
|
nullNumeric
|
||||||
|
);
|
||||||
|
type = typeFactory.createTypeWithNullability(
|
||||||
|
typeFactory.createArrayType(elementType, -1),
|
||||||
|
true
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
case COMPLEX:
|
||||||
|
type = makeComplexType(typeFactory, columnType, true);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new ISE("valueType[%s] not translatable", columnType);
|
||||||
|
} return type;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates a {@link ComplexSqlType} using the supplied {@link RelDataTypeFactory} to ensure that the
|
* Creates a {@link ComplexSqlType} using the supplied {@link RelDataTypeFactory} to ensure that the
|
||||||
* {@link ComplexSqlType} is interned. This is important because Calcite checks that the references are equal
|
* {@link ComplexSqlType} is interned. This is important because Calcite checks that the references are equal
|
||||||
|
|
|
||||||
|
|
@ -7327,4 +7327,164 @@ public class CalciteArraysQueryTest extends BaseCalciteQueryTest
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGroupByNestedArrayInline()
|
||||||
|
{
|
||||||
|
cannotVectorize();
|
||||||
|
// msq does not support nested arrays currently
|
||||||
|
msqIncompatible();
|
||||||
|
testQuery(
|
||||||
|
"SELECT c1, ARRAY_PREPEND('1', ARRAY_AGG(ARRAY[1,c2], 100000)) c5 \n"
|
||||||
|
+ "FROM (VALUES (1,1),(2,2),(3,3)) t(c1,c2)\n"
|
||||||
|
+ "GROUP BY 1 \n"
|
||||||
|
+ "HAVING ARRAY_PREPEND('1', ARRAY_AGG(ARRAY[1,c2], 100000)) <> ARRAY_PREPEND('0', ARRAY_AGG(ARRAY[1,c2], 100000))",
|
||||||
|
QUERY_CONTEXT_NO_STRINGIFY_ARRAY,
|
||||||
|
ImmutableList.of(
|
||||||
|
GroupByQuery.builder()
|
||||||
|
.setDataSource(
|
||||||
|
InlineDataSource.fromIterable(
|
||||||
|
ImmutableList.of(
|
||||||
|
new Object[]{1L, 1L},
|
||||||
|
new Object[]{2L, 2L},
|
||||||
|
new Object[]{3L, 3L}
|
||||||
|
),
|
||||||
|
RowSignature.builder()
|
||||||
|
.add("c1", ColumnType.LONG)
|
||||||
|
.add("c2", ColumnType.LONG)
|
||||||
|
.build()
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.setInterval(querySegmentSpec(Filtration.eternity()))
|
||||||
|
.setGranularity(Granularities.ALL)
|
||||||
|
.setVirtualColumns(
|
||||||
|
expressionVirtualColumn(
|
||||||
|
"v0",
|
||||||
|
"array(1,\"c2\")",
|
||||||
|
ColumnType.LONG_ARRAY
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.setDimensions(new DefaultDimensionSpec("c1", "d0", ColumnType.LONG))
|
||||||
|
.setAggregatorSpecs(
|
||||||
|
new ExpressionLambdaAggregatorFactory(
|
||||||
|
"a0",
|
||||||
|
ImmutableSet.of("v0"),
|
||||||
|
"__acc",
|
||||||
|
"ARRAY<ARRAY<LONG>>[]",
|
||||||
|
"ARRAY<ARRAY<LONG>>[]",
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
"array_append(\"__acc\", \"v0\")",
|
||||||
|
"array_concat(\"__acc\", \"a0\")",
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
HumanReadableBytes.valueOf(100000),
|
||||||
|
TestExprMacroTable.INSTANCE
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.setPostAggregatorSpecs(
|
||||||
|
expressionPostAgg(
|
||||||
|
"p0",
|
||||||
|
"array_prepend('1',\"a0\")",
|
||||||
|
ColumnType.ofArray(ColumnType.LONG_ARRAY)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.setHavingSpec(
|
||||||
|
new DimFilterHavingSpec(
|
||||||
|
expressionFilter("(array_prepend('1',\"a0\") != array_prepend('0',\"a0\"))"),
|
||||||
|
true
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.setContext(QUERY_CONTEXT_DEFAULT)
|
||||||
|
.build()
|
||||||
|
),
|
||||||
|
ImmutableList.of(
|
||||||
|
new Object[]{1, ImmutableList.of(ImmutableList.of(1L), ImmutableList.of(1L, 1L))},
|
||||||
|
new Object[]{2, ImmutableList.of(ImmutableList.of(1L), ImmutableList.of(1L, 2L))},
|
||||||
|
new Object[]{3, ImmutableList.of(ImmutableList.of(1L), ImmutableList.of(1L, 3L))}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGroupByNestedArrayInlineCount()
|
||||||
|
{
|
||||||
|
cannotVectorize();
|
||||||
|
// msq does not support nested arrays currently
|
||||||
|
msqIncompatible();
|
||||||
|
testQuery(
|
||||||
|
"SELECT COUNT(*) c FROM (\n"
|
||||||
|
+ "SELECT c1, ARRAY_PREPEND('1', ARRAY_AGG(ARRAY[1,c2], 100000)) c5 \n"
|
||||||
|
+ "FROM (VALUES (1,1),(2,2),(3,3)) t(c1,c2)\n"
|
||||||
|
+ "GROUP BY 1 \n"
|
||||||
|
+ "HAVING ARRAY_PREPEND('1', ARRAY_AGG(ARRAY[1,c2], 100000)) <> ARRAY_PREPEND('0', ARRAY_AGG(ARRAY[1,c2], 100000))\n"
|
||||||
|
+ ")",
|
||||||
|
QUERY_CONTEXT_NO_STRINGIFY_ARRAY,
|
||||||
|
ImmutableList.of(
|
||||||
|
GroupByQuery.builder()
|
||||||
|
.setDataSource(
|
||||||
|
GroupByQuery.builder()
|
||||||
|
.setDataSource(
|
||||||
|
InlineDataSource.fromIterable(
|
||||||
|
ImmutableList.of(
|
||||||
|
new Object[]{1L, 1L},
|
||||||
|
new Object[]{2L, 2L},
|
||||||
|
new Object[]{3L, 3L}
|
||||||
|
),
|
||||||
|
RowSignature.builder()
|
||||||
|
.add("c1", ColumnType.LONG)
|
||||||
|
.add("c2", ColumnType.LONG)
|
||||||
|
.build()
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.setInterval(querySegmentSpec(Filtration.eternity()))
|
||||||
|
.setGranularity(Granularities.ALL)
|
||||||
|
.setVirtualColumns(
|
||||||
|
expressionVirtualColumn(
|
||||||
|
"v0",
|
||||||
|
"array(1,\"c2\")",
|
||||||
|
ColumnType.LONG_ARRAY
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.setDimensions(new DefaultDimensionSpec("c1", "d0", ColumnType.LONG))
|
||||||
|
.setAggregatorSpecs(
|
||||||
|
new ExpressionLambdaAggregatorFactory(
|
||||||
|
"a0",
|
||||||
|
ImmutableSet.of("v0"),
|
||||||
|
"__acc",
|
||||||
|
"ARRAY<ARRAY<LONG>>[]",
|
||||||
|
"ARRAY<ARRAY<LONG>>[]",
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
"array_append(\"__acc\", \"v0\")",
|
||||||
|
"array_concat(\"__acc\", \"a0\")",
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
HumanReadableBytes.valueOf(100000),
|
||||||
|
TestExprMacroTable.INSTANCE
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.setHavingSpec(
|
||||||
|
new DimFilterHavingSpec(
|
||||||
|
expressionFilter(
|
||||||
|
"(array_prepend('1',\"a0\") != array_prepend('0',\"a0\"))"),
|
||||||
|
true
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.setContext(QUERY_CONTEXT_DEFAULT)
|
||||||
|
.build()
|
||||||
|
)
|
||||||
|
.setInterval(querySegmentSpec(Filtration.eternity()))
|
||||||
|
.setGranularity(Granularities.ALL)
|
||||||
|
.setAggregatorSpecs(new CountAggregatorFactory("_a0"))
|
||||||
|
.setContext(QUERY_CONTEXT_DEFAULT)
|
||||||
|
.build()
|
||||||
|
),
|
||||||
|
ImmutableList.of(
|
||||||
|
new Object[]{3L}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue