From f8b1f2f7f3fe6d49cbb5f19bfe6196a5b716b3ef Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 9 Mar 2020 17:48:29 -0700 Subject: [PATCH] fix issue when distinct grouping dimensions are optimized into the same virtual column expression (#9429) * fix issue when distinct grouping dimensions are optimized into the same virtual column expression * fix tests * more better * fixes --- .../hll/sql/HllSketchSqlAggregatorTest.java | 2 +- .../sql/ThetaSketchSqlAggregatorTest.java | 2 +- .../bloom/sql/BloomFilterSqlAggregator.java | 5 +- .../sql/BloomFilterSqlAggregatorTest.java | 6 +- .../aggregation/DimensionExpression.java | 55 ++++- .../druid/sql/calcite/rel/DruidQuery.java | 13 +- .../calcite/CalciteParameterQueryTest.java | 2 +- .../druid/sql/calcite/CalciteQueryTest.java | 228 ++++++++++-------- 8 files changed, 193 insertions(+), 120 deletions(-) diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java index cd18115bca7..e92b9d52fce 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java @@ -375,7 +375,7 @@ public class HllSketchSqlAggregatorTest extends CalciteTestBase Collections.singletonList( new DefaultDimensionSpec( "v0", - "v0", + "d0", ValueType.LONG ) ) diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java index ec87c57dcff..ef776162caa 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java @@ -371,7 +371,7 @@ public class ThetaSketchSqlAggregatorTest extends CalciteTestBase Collections.singletonList( new DefaultDimensionSpec( "v0", - "v0", + "d0", ValueType.LONG ) ) diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregator.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregator.java index bf17a689739..3b0df6a533b 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregator.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregator.java @@ -174,7 +174,10 @@ public class BloomFilterSqlAggregator implements SqlAggregator inputOperand.getType().getSqlTypeName() ); virtualColumns.add(virtualColumn); - spec = new DefaultDimensionSpec(virtualColumn.getOutputName(), virtualColumn.getOutputName()); + spec = new DefaultDimensionSpec( + virtualColumn.getOutputName(), + StringUtils.format("%s:%s", name, virtualColumn.getOutputName()) + ); } aggregatorFactory = new BloomFilterAggregatorFactory( diff --git a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java index 1e875f2f012..ef2f4d1bbe5 100644 --- a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java +++ b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java @@ -529,7 +529,7 @@ public class BloomFilterSqlAggregatorTest extends InitializedNullHandlingTest ImmutableList.of( new BloomFilterAggregatorFactory( "a0:agg", - new DefaultDimensionSpec("v0", "v0"), + new DefaultDimensionSpec("v0", "a0:v0"), TEST_NUM_ENTRIES ) ) @@ -597,7 +597,7 @@ public class BloomFilterSqlAggregatorTest extends InitializedNullHandlingTest ImmutableList.of( new BloomFilterAggregatorFactory( "a0:agg", - new DefaultDimensionSpec("v0", "v0"), + new DefaultDimensionSpec("v0", "a0:v0"), TEST_NUM_ENTRIES ) ) @@ -665,7 +665,7 @@ public class BloomFilterSqlAggregatorTest extends InitializedNullHandlingTest ImmutableList.of( new BloomFilterAggregatorFactory( "a0:agg", - new DefaultDimensionSpec("v0", "v0"), + new DefaultDimensionSpec("v0", "a0:v0"), TEST_NUM_ENTRIES ) ) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/DimensionExpression.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/DimensionExpression.java index 72f296e3158..10b5bbefe34 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/DimensionExpression.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/DimensionExpression.java @@ -19,6 +19,7 @@ package org.apache.druid.sql.calcite.aggregation; +import com.google.common.base.Preconditions; import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.segment.column.ValueType; @@ -28,21 +29,55 @@ import java.util.Objects; public class DimensionExpression { - private final String outputName; - private final DruidExpression expression; - private final ValueType outputType; - - public DimensionExpression( + /** + * Create a dimension expression for direct column access or simple extractions + */ + public static DimensionExpression ofSimpleColumn( final String outputName, final DruidExpression expression, final ValueType outputType ) { + return new DimensionExpression(outputName, outputName, expression, outputType); + } + + /** + * Create a dimension expression for a virtual column + */ + public static DimensionExpression ofVirtualColumn( + final String virtualColumn, + final String outputName, + final DruidExpression expression, + final ValueType outputType + ) + { + return new DimensionExpression(virtualColumn, outputName, expression, outputType); + } + + private final String virtualColumn; + private final String outputName; + private final DruidExpression expression; + private final ValueType outputType; + + private DimensionExpression( + final String virtualColumn, + final String outputName, + final DruidExpression expression, + final ValueType outputType + ) + { + Preconditions.checkArgument(!expression.isSimpleExtraction() || outputName.equals(virtualColumn)); + this.virtualColumn = virtualColumn; this.outputName = outputName; this.expression = expression; this.outputType = outputType; } + public String getVirtualColumn() + { + return virtualColumn; + } + public String getOutputName() { return outputName; @@ -58,7 +93,7 @@ public class DimensionExpression if (expression.isSimpleExtraction()) { return expression.getSimpleExtraction().toDimensionSpec(outputName, outputType); } else { - return new DefaultDimensionSpec(getOutputName(), getOutputName(), outputType); + return new DefaultDimensionSpec(virtualColumn, outputName, outputType); } } @@ -72,7 +107,8 @@ public class DimensionExpression return false; } final DimensionExpression that = (DimensionExpression) o; - return Objects.equals(outputName, that.outputName) && + return Objects.equals(virtualColumn, that.virtualColumn) && + Objects.equals(outputName, that.outputName) && Objects.equals(expression, that.expression) && outputType == that.outputType; } @@ -80,14 +116,15 @@ public class DimensionExpression @Override public int hashCode() { - return Objects.hash(outputName, expression, outputType); + return Objects.hash(virtualColumn, outputName, expression, outputType); } @Override public String toString() { return "DimensionExpression{" + - "outputName='" + outputName + '\'' + + "virtualColumn='" + virtualColumn + '\'' + + ", outputName='" + outputName + '\'' + ", expression=" + expression + ", outputType=" + outputType + '}'; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java index 09bc584b774..dfb3f1669ae 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java @@ -401,19 +401,18 @@ public class DruidQuery final VirtualColumn virtualColumn; - final String dimOutputName; + + final String dimOutputName = outputNamePrefix + outputNameCounter++; if (!druidExpression.isSimpleExtraction()) { virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, druidExpression, sqlTypeName ); - dimOutputName = virtualColumn.getOutputName(); + dimensions.add(DimensionExpression.ofVirtualColumn(virtualColumn.getOutputName(), dimOutputName, druidExpression, outputType)); } else { - dimOutputName = outputNamePrefix + outputNameCounter++; + dimensions.add(DimensionExpression.ofSimpleColumn(dimOutputName, druidExpression, outputType)); } - - dimensions.add(new DimensionExpression(dimOutputName, druidExpression, outputType)); } return dimensions; @@ -623,8 +622,8 @@ public class DruidQuery if (grouping != null) { if (includeDimensions) { for (DimensionExpression expression : grouping.getDimensions()) { - if (virtualColumnRegistry.isVirtualColumnDefined(expression.getOutputName())) { - virtualColumns.add(virtualColumnRegistry.getVirtualColumn(expression.getOutputName())); + if (virtualColumnRegistry.isVirtualColumnDefined(expression.getVirtualColumn())) { + virtualColumns.add(virtualColumnRegistry.getVirtualColumn(expression.getVirtualColumn())); } } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteParameterQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteParameterQueryTest.java index 98263cb2dea..87e8d2947cb 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteParameterQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteParameterQueryTest.java @@ -636,7 +636,7 @@ public class CalciteParameterQueryTest extends BaseCalciteQueryTest ValueType.STRING ) ) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0", ValueType.STRING))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ValueType.STRING))) .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))) .setContext(QUERY_CONTEXT_DEFAULT) .build() diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index e230635476f..76cd55b72e7 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -3146,7 +3146,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ValueType.STRING ) ) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0"))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0"))) .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))) .setContext(QUERY_CONTEXT_DEFAULT) .build() @@ -3183,7 +3183,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ValueType.STRING ) ) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0"))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0"))) .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))) .setContext(QUERY_CONTEXT_DEFAULT) .build() @@ -3568,7 +3568,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ValueType.STRING ) ) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0", ValueType.STRING))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ValueType.STRING))) .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))) .setContext(QUERY_CONTEXT_DEFAULT) .build() @@ -4617,13 +4617,13 @@ public class CalciteQueryTest extends BaseCalciteQueryTest expressionVirtualColumn("v0", "(floor((\"m1\" / 2)) * 2)", ValueType.FLOAT) ) .setDimFilter(bound("v0", "-1", null, true, false, null, StringComparators.NUMERIC)) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0", ValueType.FLOAT))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ValueType.FLOAT))) .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))) .setLimitSpec( new DefaultLimitSpec( ImmutableList.of( new OrderByColumnSpec( - "v0", + "d0", OrderByColumnSpec.Direction.DESCENDING, StringComparators.NUMERIC ) @@ -4668,13 +4668,13 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .setDimFilter( bound("v0", "-1", null, true, false, null, StringComparators.NUMERIC) ) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0", ValueType.LONG))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ValueType.LONG))) .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))) .setLimitSpec( new DefaultLimitSpec( ImmutableList.of( new OrderByColumnSpec( - "v0", + "d0", OrderByColumnSpec.Direction.DESCENDING, StringComparators.NUMERIC ) @@ -4723,13 +4723,13 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .setDimFilter( bound("v0", "-1", null, true, false, null, StringComparators.NUMERIC) ) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0", ValueType.FLOAT))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ValueType.FLOAT))) .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))) .setLimitSpec( new DefaultLimitSpec( ImmutableList.of( new OrderByColumnSpec( - "v0", + "d0", OrderByColumnSpec.Direction.DESCENDING, StringComparators.NUMERIC ) @@ -5478,7 +5478,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest StringComparators.NUMERIC ) ) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0", ValueType.LONG))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ValueType.LONG))) .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))) .setContext(QUERY_CONTEXT_DEFAULT) .build() @@ -6045,8 +6045,8 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ) ) .setDimensions(dimensions( - new DefaultDimensionSpec("v0", "v0", ValueType.LONG), - new DefaultDimensionSpec("d0", "_d0", ValueType.STRING) + new DefaultDimensionSpec("v0", "_d0", ValueType.LONG), + new DefaultDimensionSpec("d0", "_d1", ValueType.STRING) )) .setAggregatorSpecs( aggregators( @@ -6229,7 +6229,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ) .setDimensions(dimensions(new DefaultDimensionSpec( "v0", - "v0", + "d0", ValueType.LONG ))) .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))) @@ -6244,7 +6244,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest new LongMinAggregatorFactory("_a1", "a0"), new LongSumAggregatorFactory("_a2:sum", "a0"), new CountAggregatorFactory("_a2:count"), - new LongMaxAggregatorFactory("_a3", "v0"), + new LongMaxAggregatorFactory("_a3", "d0"), new CountAggregatorFactory("_a4") )) .setPostAggregatorSpecs( @@ -6301,7 +6301,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ) .setDimensions(dimensions(new DefaultDimensionSpec( "v0", - "v0", + "d0", ValueType.LONG ))) .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))) @@ -6375,7 +6375,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ) .setDimensions(dimensions(new DefaultDimensionSpec( "v0", - "v0", + "d0", ValueType.LONG ))) .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))) @@ -6733,7 +6733,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ) .setDimensions(dimensions(new DefaultDimensionSpec( "v0", - "v0", + "d0", ValueType.LONG ))) .setAggregatorSpecs( @@ -7490,7 +7490,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest "(((timestamp_extract(\"__time\",'MONTH','UTC') - 1) / 3) + 1)", ValueType.LONG )) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0", ValueType.LONG))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ValueType.LONG))) .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))) .setContext(QUERY_CONTEXT_DEFAULT) .build() @@ -8053,7 +8053,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .setVirtualColumns( expressionVirtualColumn("v0", "floor(CAST(\"dim1\", 'DOUBLE'))", ValueType.FLOAT) ) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0", ValueType.FLOAT))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ValueType.FLOAT))) .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))) .setContext(QUERY_CONTEXT_DEFAULT) .build() @@ -8091,7 +8091,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest dimensions( new DefaultDimensionSpec( "v0", - "v0", + "d0", ValueType.FLOAT ) ) @@ -8101,7 +8101,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest new DefaultLimitSpec( ImmutableList.of( new OrderByColumnSpec( - "v0", + "d0", OrderByColumnSpec.Direction.DESCENDING, StringComparators.NUMERIC ) @@ -8146,8 +8146,8 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0", ValueType.LONG), - new DefaultDimensionSpec("dim2", "d0") + new DefaultDimensionSpec("v0", "d0", ValueType.LONG), + new DefaultDimensionSpec("dim2", "d1") ) ) .setAggregatorSpecs( @@ -8159,12 +8159,12 @@ public class CalciteQueryTest extends BaseCalciteQueryTest new DefaultLimitSpec( ImmutableList.of( new OrderByColumnSpec( - "v0", + "d0", OrderByColumnSpec.Direction.ASCENDING, StringComparators.NUMERIC ), new OrderByColumnSpec( - "d0", + "d1", OrderByColumnSpec.Direction.ASCENDING, StringComparators.LEXICOGRAPHIC ), @@ -8213,7 +8213,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .setInterval(querySegmentSpec(Filtration.eternity())) .setGranularity(Granularities.ALL) .setVirtualColumns(expressionVirtualColumn("v0", "strlen(\"dim1\")", ValueType.LONG)) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0", ValueType.LONG))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ValueType.LONG))) .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))) .setContext(QUERY_CONTEXT_DEFAULT) .build() @@ -8969,13 +8969,13 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ValueType.LONG ) ) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0", ValueType.LONG))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ValueType.LONG))) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) .setLimitSpec( new DefaultLimitSpec( ImmutableList.of( new OrderByColumnSpec( - "v0", + "d0", OrderByColumnSpec.Direction.ASCENDING, StringComparators.NUMERIC ) @@ -9020,13 +9020,13 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ValueType.LONG ) ) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0", ValueType.LONG))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ValueType.LONG))) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) .setLimitSpec( new DefaultLimitSpec( ImmutableList.of( new OrderByColumnSpec( - "v0", + "d0", OrderByColumnSpec.Direction.ASCENDING, StringComparators.NUMERIC ) @@ -9303,13 +9303,13 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ValueType.LONG ) ) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0", ValueType.LONG))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ValueType.LONG))) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) .setLimitSpec( new DefaultLimitSpec( ImmutableList.of( new OrderByColumnSpec( - "v0", + "d0", OrderByColumnSpec.Direction.ASCENDING, StringComparators.NUMERIC ) @@ -9352,13 +9352,13 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ValueType.STRING ) ) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0", ValueType.STRING))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ValueType.STRING))) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) .setLimitSpec( new DefaultLimitSpec( ImmutableList.of( new OrderByColumnSpec( - "v0", + "d0", OrderByColumnSpec.Direction.ASCENDING, StringComparators.LEXICOGRAPHIC ) @@ -9399,7 +9399,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ValueType.LONG ) ) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0", ValueType.LONG))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ValueType.LONG))) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) .setContext(QUERY_CONTEXT_DEFAULT) .build() @@ -9437,7 +9437,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ValueType.LONG ) ) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0", ValueType.LONG))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ValueType.LONG))) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) .setContext(QUERY_CONTEXT_LOS_ANGELES) .build() @@ -9561,7 +9561,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .setDimensions( dimensions( new DefaultDimensionSpec("dim2", "d0"), - new DefaultDimensionSpec("v0", "v0", ValueType.LONG) + new DefaultDimensionSpec("v0", "d1", ValueType.LONG) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) @@ -9570,7 +9570,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ImmutableList.of( new OrderByColumnSpec("d0", OrderByColumnSpec.Direction.ASCENDING), new OrderByColumnSpec( - "v0", + "d1", OrderByColumnSpec.Direction.ASCENDING, StringComparators.NUMERIC ) @@ -9629,16 +9629,16 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0"), - new DefaultDimensionSpec("v1", "v1", ValueType.LONG) + new DefaultDimensionSpec("v0", "d0"), + new DefaultDimensionSpec("v1", "d1", ValueType.LONG) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) .setSubtotalsSpec( ImmutableList.of( - ImmutableList.of("v0", "v1"), - ImmutableList.of("v0"), - ImmutableList.of("v1"), + ImmutableList.of("d0", "d1"), + ImmutableList.of("d0"), + ImmutableList.of("d1"), ImmutableList.of() ) ) @@ -9720,15 +9720,15 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0"), - new DefaultDimensionSpec("v1", "v1", ValueType.LONG) + new DefaultDimensionSpec("v0", "d0"), + new DefaultDimensionSpec("v1", "d1", ValueType.LONG) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) .setSubtotalsSpec( ImmutableList.of( - ImmutableList.of("v0", "v1"), - ImmutableList.of("v0"), + ImmutableList.of("d0", "d1"), + ImmutableList.of("d0"), ImmutableList.of() ) ) @@ -9779,15 +9779,15 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0", ValueType.LONG), - new DefaultDimensionSpec("v1", "v1") + new DefaultDimensionSpec("v0", "d0", ValueType.LONG), + new DefaultDimensionSpec("v1", "d1") ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) .setSubtotalsSpec( ImmutableList.of( - ImmutableList.of("v0", "v1"), - ImmutableList.of("v0"), + ImmutableList.of("d0", "d1"), + ImmutableList.of("d0"), ImmutableList.of() ) ) @@ -9836,16 +9836,16 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0"), - new DefaultDimensionSpec("v1", "v1", ValueType.LONG) + new DefaultDimensionSpec("v0", "d0"), + new DefaultDimensionSpec("v1", "d1", ValueType.LONG) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) .setSubtotalsSpec( ImmutableList.of( - ImmutableList.of("v0", "v1"), - ImmutableList.of("v0"), - ImmutableList.of("v1"), + ImmutableList.of("d0", "d1"), + ImmutableList.of("d0"), + ImmutableList.of("d1"), ImmutableList.of() ) ) @@ -9897,17 +9897,17 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0"), - new DefaultDimensionSpec("v2", "v2", ValueType.LONG) + new DefaultDimensionSpec("v0", "d0"), + new DefaultDimensionSpec("v2", "d2", ValueType.LONG) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) .setSubtotalsSpec( ImmutableList.of( - ImmutableList.of("v0", "v2"), - ImmutableList.of("v0"), + ImmutableList.of("d0", "d2"), + ImmutableList.of("d0"), ImmutableList.of(), - ImmutableList.of("v2") + ImmutableList.of("d2") ) ) .setContext(QUERY_CONTEXT_DEFAULT) @@ -9959,15 +9959,15 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0"), - new DefaultDimensionSpec("v1", "v1", ValueType.LONG) + new DefaultDimensionSpec("v0", "d0"), + new DefaultDimensionSpec("v1", "d1", ValueType.LONG) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) .setSubtotalsSpec( ImmutableList.of( - ImmutableList.of("v0"), - ImmutableList.of("v1"), + ImmutableList.of("d0"), + ImmutableList.of("d1"), ImmutableList.of() ) ) @@ -10015,15 +10015,15 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0"), - new DefaultDimensionSpec("v1", "v1", ValueType.LONG) + new DefaultDimensionSpec("v0", "d0"), + new DefaultDimensionSpec("v1", "d1", ValueType.LONG) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) .setSubtotalsSpec( ImmutableList.of( - ImmutableList.of("v0"), - ImmutableList.of("v1"), + ImmutableList.of("d0"), + ImmutableList.of("d1"), ImmutableList.of() ) ) @@ -10031,12 +10031,12 @@ public class CalciteQueryTest extends BaseCalciteQueryTest new DefaultLimitSpec( ImmutableList.of( new OrderByColumnSpec( - "v1", + "d1", Direction.ASCENDING, StringComparators.NUMERIC ), new OrderByColumnSpec( - "v0", + "d0", Direction.DESCENDING, StringComparators.LEXICOGRAPHIC ) @@ -10088,15 +10088,15 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0"), - new DefaultDimensionSpec("v1", "v1", ValueType.LONG) + new DefaultDimensionSpec("v0", "d0"), + new DefaultDimensionSpec("v1", "d1", ValueType.LONG) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) .setSubtotalsSpec( ImmutableList.of( - ImmutableList.of("v0"), - ImmutableList.of("v1"), + ImmutableList.of("d0"), + ImmutableList.of("d1"), ImmutableList.of() ) ) @@ -10157,15 +10157,15 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0"), - new DefaultDimensionSpec("v1", "v1", ValueType.LONG) + new DefaultDimensionSpec("v0", "d0"), + new DefaultDimensionSpec("v1", "d1", ValueType.LONG) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) .setSubtotalsSpec( ImmutableList.of( - ImmutableList.of("v0"), - ImmutableList.of("v1"), + ImmutableList.of("d0"), + ImmutableList.of("d1"), ImmutableList.of() ) ) @@ -10457,7 +10457,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest selector("dim2", "abc", null) ) ) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0", ValueType.LONG))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ValueType.LONG))) .setInterval(querySegmentSpec(Filtration.eternity())) .setGranularity(Granularities.ALL) .setAggregatorSpecs( @@ -10477,7 +10477,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest new DefaultLimitSpec( ImmutableList.of( new OrderByColumnSpec( - "v0", + "d0", OrderByColumnSpec.Direction.ASCENDING, StringComparators.NUMERIC ) @@ -11444,7 +11444,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ValueType.STRING ) ) - .setDimensions(dimensions(new DefaultDimensionSpec("v0", "v0", ValueType.STRING))) + .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ValueType.STRING))) .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))) .setContext(QUERY_CONTEXT_DEFAULT) .build() @@ -11501,7 +11501,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .setVirtualColumns(expressionVirtualColumn("v0", "concat(\"dim3\",'foo')", ValueType.STRING)) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0", ValueType.STRING) + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) @@ -11536,7 +11536,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .setVirtualColumns(expressionVirtualColumn("v0", "concat(\"dim3\",'foo')", ValueType.STRING)) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0", ValueType.STRING) + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) ) ) .setDimFilter(selector("v0", "bfoo", null)) @@ -11820,13 +11820,13 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .setDimensions( dimensions( new DefaultDimensionSpec("dim1", "_d0", ValueType.STRING), - new DefaultDimensionSpec("v0", "v0", ValueType.LONG) + new DefaultDimensionSpec("v0", "_d1", ValueType.LONG) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) .setLimitSpec(new DefaultLimitSpec( ImmutableList.of(new OrderByColumnSpec( - "v0", + "_d1", Direction.DESCENDING, StringComparators.NUMERIC )), @@ -11883,7 +11883,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .setVirtualColumns(expressionVirtualColumn("v0", "array_append(\"dim3\",'foo')", ValueType.STRING)) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0", ValueType.STRING) + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) @@ -11939,7 +11939,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .setVirtualColumns(expressionVirtualColumn("v0", "array_prepend('foo',\"dim3\")", ValueType.STRING)) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0", ValueType.STRING) + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) @@ -11994,8 +11994,8 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0", ValueType.STRING), - new DefaultDimensionSpec("v1", "v1", ValueType.STRING) + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING), + new DefaultDimensionSpec("v1", "_d1", ValueType.STRING) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) @@ -12049,7 +12049,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .setVirtualColumns(expressionVirtualColumn("v0", "array_concat(\"dim3\",\"dim3\")", ValueType.STRING)) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0", ValueType.STRING) + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) @@ -12084,7 +12084,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .setVirtualColumns(expressionVirtualColumn("v0", "array_offset(\"dim3\",1)", ValueType.STRING)) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0", ValueType.STRING) + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) @@ -12123,7 +12123,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .setVirtualColumns(expressionVirtualColumn("v0", "array_ordinal(\"dim3\",2)", ValueType.STRING)) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0", ValueType.STRING) + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) @@ -12162,7 +12162,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .setVirtualColumns(expressionVirtualColumn("v0", "array_offset_of(\"dim3\",'b')", ValueType.LONG)) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0", ValueType.LONG) + new DefaultDimensionSpec("v0", "_d0", ValueType.LONG) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) @@ -12201,7 +12201,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .setVirtualColumns(expressionVirtualColumn("v0", "array_ordinal_of(\"dim3\",'b')", ValueType.LONG)) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0", ValueType.LONG) + new DefaultDimensionSpec("v0", "_d0", ValueType.LONG) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) @@ -12257,7 +12257,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .setVirtualColumns(expressionVirtualColumn("v0", "array_to_string(\"dim3\",',')", ValueType.STRING)) .setDimensions( dimensions( - new DefaultDimensionSpec("v0", "v0", ValueType.STRING) + new DefaultDimensionSpec("v0", "_d0", ValueType.STRING) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) @@ -12319,7 +12319,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .setDimFilter(bound("v0", "0", null, true, false, null, StringComparators.NUMERIC)) .setDimensions( dimensions( - new DefaultDimensionSpec("v1", "v1", ValueType.STRING) + new DefaultDimensionSpec("v1", "_d0", ValueType.STRING) ) ) .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) @@ -12461,4 +12461,38 @@ public class CalciteQueryTest extends BaseCalciteQueryTest results2 ); } + + @Test + public void testRepeatedIdenticalVirtualExpressionGrouping() throws Exception + { + cannotVectorize(); + + final String query = "SELECT \n" + + "\tCASE dim1 WHEN NULL THEN FALSE ELSE TRUE END AS col_a,\n" + + "\tCASE dim2 WHEN NULL THEN FALSE ELSE TRUE END AS col_b\n" + + "FROM foo\n" + + "GROUP BY 1, 2"; + + testQuery( + query, + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE1) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setVirtualColumns(expressionVirtualColumn("v0", "1", ValueType.LONG)) + .setDimensions( + dimensions( + new DefaultDimensionSpec("v0", "d0", ValueType.LONG), + new DefaultDimensionSpec("v0", "d1", ValueType.LONG) + ) + ) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{true, true} + ) + ); + } }