diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java index e0f50b0a7e2..fc737cd30ad 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java @@ -38,6 +38,7 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; import it.unimi.dsi.fastutil.ints.IntArraySet; import it.unimi.dsi.fastutil.ints.IntList; import it.unimi.dsi.fastutil.ints.IntSet; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.common.guava.FutureUtils; import org.apache.druid.data.input.StringTuple; import org.apache.druid.data.input.impl.DimensionSchema; @@ -185,6 +186,7 @@ import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import org.apache.druid.segment.transform.TransformSpec; import org.apache.druid.server.DruidNode; import org.apache.druid.sql.calcite.rel.DruidQuery; +import org.apache.druid.sql.calcite.run.SqlResults; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentTimeline; import org.apache.druid.timeline.partition.DimensionRangeShardSpec; @@ -585,14 +587,10 @@ public class ControllerImpl implements Controller QueryValidator.validateQueryDef(queryDef); queryDefRef.set(queryDef); - long maxParseExceptions = -1; - - if (task.getSqlQueryContext() != null) { - maxParseExceptions = Optional.ofNullable( - task.getSqlQueryContext().get(MSQWarnings.CTX_MAX_PARSE_EXCEPTIONS_ALLOWED)) - .map(DimensionHandlerUtils::convertObjectToLong) - .orElse(MSQWarnings.DEFAULT_MAX_PARSE_EXCEPTIONS_ALLOWED); - } + final long maxParseExceptions = task.getQuerySpec().getQuery().context().getLong( + MSQWarnings.CTX_MAX_PARSE_EXCEPTIONS_ALLOWED, + MSQWarnings.DEFAULT_MAX_PARSE_EXCEPTIONS_ALLOWED + ); ImmutableMap.Builder taskContextOverridesBuilder = ImmutableMap.builder(); taskContextOverridesBuilder @@ -1435,15 +1433,26 @@ public class ControllerImpl implements Controller .stream() .map( mapping -> - columnSelectorFactory.makeColumnValueSelector( - mapping.getQueryColumn()) + columnSelectorFactory.makeColumnValueSelector(mapping.getQueryColumn()) ).collect(Collectors.toList()); + final List sqlTypeNames = task.getSqlTypeNames(); final List retVal = new ArrayList<>(); while (!cursor.isDone()) { final Object[] row = new Object[columnMappings.size()]; for (int i = 0; i < row.length; i++) { - row[i] = selectors.get(i).getObject(); + final Object value = selectors.get(i).getObject(); + if (sqlTypeNames == null || task.getSqlResultsContext() == null) { + // SQL type unknown, or no SQL results context: pass-through as is. + row[i] = value; + } else { + row[i] = SqlResults.coerce( + context.jsonMapper(), + task.getSqlResultsContext(), + value, + sqlTypeNames.get(i) + ); + } } retVal.add(row); cursor.advance(); @@ -1998,7 +2007,7 @@ public class ControllerImpl implements Controller final QueryDefinition queryDef, final Yielder resultsYielder, final ColumnMappings columnMappings, - @Nullable final List sqlTypeNames + @Nullable final List sqlTypeNames ) { final RowSignature querySignature = queryDef.getFinalStageDefinition().getSignature(); diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/KeyStatisticsCollectionProcessor.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/KeyStatisticsCollectionProcessor.java index af528f6ecbe..f9832670801 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/KeyStatisticsCollectionProcessor.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/KeyStatisticsCollectionProcessor.java @@ -36,7 +36,6 @@ import org.apache.druid.frame.processor.ReturnOrAwait; import org.apache.druid.frame.read.FrameReader; import org.apache.druid.frame.read.FrameReaderUtils; import org.apache.druid.java.util.common.ISE; -import org.apache.druid.msq.sql.MSQTaskQueryMaker; import org.apache.druid.msq.statistics.ClusterByStatisticsCollector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.Cursor; @@ -51,14 +50,14 @@ public class KeyStatisticsCollectionProcessor implements FrameProcessor sqlQueryContext; - // Enables users, and the web console, to see the original SQL type names (if any). Not used by any other Druid logic. + /** + * Enables usage of {@link SqlResults#coerce(ObjectMapper, SqlResults.Context, Object, SqlTypeName)}. + */ @Nullable - private final List sqlTypeNames; + private final SqlResults.Context sqlResultsContext; + + /** + * SQL type names for each field in the resultset. + */ + @Nullable + private final List sqlTypeNames; // Using an Injector directly because tasks do not have a way to provide their own Guice modules. @JacksonInject @@ -88,7 +103,8 @@ public class MSQControllerTask extends AbstractTask @JsonProperty("spec") MSQSpec querySpec, @JsonProperty("sqlQuery") @Nullable String sqlQuery, @JsonProperty("sqlQueryContext") @Nullable Map sqlQueryContext, - @JsonProperty("sqlTypeNames") @Nullable List sqlTypeNames, + @JsonProperty("sqlResultsContext") @Nullable SqlResults.Context sqlResultsContext, + @JsonProperty("sqlTypeNames") @Nullable List sqlTypeNames, @JsonProperty("context") @Nullable Map context ) { @@ -103,6 +119,7 @@ public class MSQControllerTask extends AbstractTask this.querySpec = querySpec; this.sqlQuery = sqlQuery; this.sqlQueryContext = sqlQueryContext; + this.sqlResultsContext = sqlResultsContext; this.sqlTypeNames = sqlTypeNames; addToContext(Tasks.FORCE_TIME_CHUNK_LOCK_KEY, true); @@ -132,7 +149,15 @@ public class MSQControllerTask extends AbstractTask @Nullable @JsonProperty @JsonInclude(JsonInclude.Include.NON_NULL) - public String getSqlQuery() + public List getSqlTypeNames() + { + return sqlTypeNames; + } + + @Nullable + @JsonProperty + @JsonInclude(JsonInclude.Include.NON_NULL) + private String getSqlQuery() { return sqlQuery; } @@ -140,7 +165,7 @@ public class MSQControllerTask extends AbstractTask @Nullable @JsonProperty @JsonInclude(JsonInclude.Include.NON_NULL) - public Map getSqlQueryContext() + private Map getSqlQueryContext() { return sqlQueryContext; } @@ -148,9 +173,9 @@ public class MSQControllerTask extends AbstractTask @Nullable @JsonProperty @JsonInclude(JsonInclude.Include.NON_NULL) - public List getSqlTypeNames() + public SqlResults.Context getSqlResultsContext() { - return sqlTypeNames; + return sqlResultsContext; } @Override diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQSpec.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQSpec.java index b5f616b9640..904ab662df3 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQSpec.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQSpec.java @@ -26,6 +26,7 @@ import org.apache.druid.msq.kernel.WorkerAssignmentStrategy; import org.apache.druid.query.Query; import javax.annotation.Nullable; +import java.util.Map; import java.util.Objects; public class MSQSpec @@ -87,6 +88,21 @@ public class MSQSpec return tuningConfig; } + public MSQSpec withOverriddenContext(Map contextOverride) + { + if (contextOverride == null || contextOverride.isEmpty()) { + return this; + } else { + return new MSQSpec( + query.withOverriddenContext(contextOverride), + columnMappings, + destination, + assignmentStrategy, + tuningConfig + ); + } + } + @Override public boolean equals(Object o) { diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQResultsReport.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQResultsReport.java index 58911389b36..c145ae77d57 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQResultsReport.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQResultsReport.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.common.guava.Yielder; import org.apache.druid.java.util.common.guava.Yielders; @@ -40,12 +41,12 @@ public class MSQResultsReport */ private final List signature; @Nullable - private final List sqlTypeNames; + private final List sqlTypeNames; private final Yielder resultYielder; public MSQResultsReport( final List signature, - @Nullable final List sqlTypeNames, + @Nullable final List sqlTypeNames, final Yielder resultYielder ) { @@ -60,7 +61,7 @@ public class MSQResultsReport @JsonCreator static MSQResultsReport fromJson( @JsonProperty("signature") final List signature, - @JsonProperty("sqlTypeNames") @Nullable final List sqlTypeNames, + @JsonProperty("sqlTypeNames") @Nullable final List sqlTypeNames, @JsonProperty("results") final List results ) { @@ -76,7 +77,7 @@ public class MSQResultsReport @Nullable @JsonProperty("sqlTypeNames") @JsonInclude(JsonInclude.Include.NON_NULL) - public List getSqlTypeNames() + public List getSqlTypeNames() { return sqlTypeNames; } diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java index 783b3714fc0..1d5d1f522d6 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java @@ -58,6 +58,7 @@ import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.rel.DruidQuery; import org.apache.druid.sql.calcite.rel.Grouping; import org.apache.druid.sql.calcite.run.QueryMaker; +import org.apache.druid.sql.calcite.run.SqlResults; import org.apache.druid.sql.calcite.table.RowSignatures; import org.joda.time.Interval; @@ -77,11 +78,6 @@ public class MSQTaskQueryMaker implements QueryMaker private static final String DESTINATION_REPORT = "taskReport"; private static final Granularity DEFAULT_SEGMENT_GRANULARITY = Granularities.ALL; - private static final int DEFAULT_ROWS_PER_SEGMENT = 3000000; - - // Lower than the default to minimize the impact of per-row overheads that are not accounted for by - // OnheapIncrementalIndex. For example: overheads related to creating bitmaps during persist. - private static final int DEFAULT_ROWS_IN_MEMORY = 100000; private final String targetDataSource; private final OverlordClient overlordClient; @@ -111,14 +107,21 @@ public class MSQTaskQueryMaker implements QueryMaker Hook.QUERY_PLAN.run(druidQuery.getQuery()); String taskId = MSQTasks.controllerTaskId(plannerContext.getSqlQueryId()); - QueryContext queryContext = plannerContext.queryContext(); - String msqMode = MultiStageQueryContext.getMSQMode(queryContext); + // SQL query context: context provided by the user, and potentially modified by handlers during planning. + // Does not directly influence task execution, but it does form the basis for the initial native query context, + // which *does* influence task execution. + final QueryContext sqlQueryContext = plannerContext.queryContext(); + + // Native query context: sqlQueryContext plus things that we add prior to creating a controller task. + final Map nativeQueryContext = new HashMap<>(sqlQueryContext.asMap()); + + final String msqMode = MultiStageQueryContext.getMSQMode(sqlQueryContext); if (msqMode != null) { - MSQMode.populateDefaultQueryContext(msqMode, plannerContext.queryContextMap()); + MSQMode.populateDefaultQueryContext(msqMode, nativeQueryContext); } final String ctxDestination = - DimensionHandlerUtils.convertObjectToString(MultiStageQueryContext.getDestination(queryContext)); + DimensionHandlerUtils.convertObjectToString(MultiStageQueryContext.getDestination(sqlQueryContext)); Object segmentGranularity; try { @@ -131,7 +134,7 @@ public class MSQTaskQueryMaker implements QueryMaker + "segment graularity"); } - final int maxNumTasks = MultiStageQueryContext.getMaxNumTasks(queryContext); + final int maxNumTasks = MultiStageQueryContext.getMaxNumTasks(sqlQueryContext); if (maxNumTasks < 2) { throw new IAE(MultiStageQueryContext.CTX_MAX_NUM_TASKS @@ -140,23 +143,13 @@ public class MSQTaskQueryMaker implements QueryMaker // This parameter is used internally for the number of worker tasks only, so we subtract 1 final int maxNumWorkers = maxNumTasks - 1; - - final int rowsPerSegment = MultiStageQueryContext.getRowsPerSegment( - queryContext, - DEFAULT_ROWS_PER_SEGMENT - ); - - final int maxRowsInMemory = MultiStageQueryContext.getRowsInMemory( - queryContext, - DEFAULT_ROWS_IN_MEMORY - ); - - final IndexSpec indexSpec = MultiStageQueryContext.getIndexSpec(queryContext, jsonMapper); - - final boolean finalizeAggregations = MultiStageQueryContext.isFinalizeAggregations(queryContext); + final int rowsPerSegment = MultiStageQueryContext.getRowsPerSegment(sqlQueryContext); + final int maxRowsInMemory = MultiStageQueryContext.getRowsInMemory(sqlQueryContext); + final IndexSpec indexSpec = MultiStageQueryContext.getIndexSpec(sqlQueryContext, jsonMapper); + final boolean finalizeAggregations = MultiStageQueryContext.isFinalizeAggregations(sqlQueryContext); final List replaceTimeChunks = - Optional.ofNullable(plannerContext.queryContext().get(DruidSqlReplace.SQL_REPLACE_TIME_CHUNKS)) + Optional.ofNullable(sqlQueryContext.get(DruidSqlReplace.SQL_REPLACE_TIME_CHUNKS)) .map( s -> { if (s instanceof String && "all".equals(StringUtils.toLowerCase((String) s))) { @@ -179,7 +172,7 @@ public class MSQTaskQueryMaker implements QueryMaker final Map aggregationIntermediateTypeMap = finalizeAggregations ? null /* Not needed */ : buildAggregationIntermediateTypeMap(druidQuery); - final List sqlTypeNames = new ArrayList<>(); + final List sqlTypeNames = new ArrayList<>(); final List columnMappings = new ArrayList<>(); for (final Pair entry : fieldMapping) { @@ -195,7 +188,7 @@ public class MSQTaskQueryMaker implements QueryMaker sqlTypeName = druidQuery.getOutputRowType().getFieldList().get(entry.getKey()).getType().getSqlTypeName(); } - sqlTypeNames.add(sqlTypeName.getName()); + sqlTypeNames.add(sqlTypeName); columnMappings.add(new ColumnMapping(queryColumn, outputColumns)); } @@ -214,7 +207,7 @@ public class MSQTaskQueryMaker implements QueryMaker throw new ISE("Unable to convert %s to a segment granularity", segmentGranularity); } - final List segmentSortOrder = MultiStageQueryContext.getSortOrder(queryContext); + final List segmentSortOrder = MultiStageQueryContext.getSortOrder(sqlQueryContext); MSQTaskQueryMakerUtils.validateSegmentSortOrder( segmentSortOrder, @@ -245,15 +238,16 @@ public class MSQTaskQueryMaker implements QueryMaker .query(druidQuery.getQuery().withOverriddenContext(nativeQueryContextOverrides)) .columnMappings(new ColumnMappings(columnMappings)) .destination(destination) - .assignmentStrategy(MultiStageQueryContext.getAssignmentStrategy(queryContext)) + .assignmentStrategy(MultiStageQueryContext.getAssignmentStrategy(sqlQueryContext)) .tuningConfig(new MSQTuningConfig(maxNumWorkers, maxRowsInMemory, rowsPerSegment, indexSpec)) .build(); final MSQControllerTask controllerTask = new MSQControllerTask( taskId, - querySpec, + querySpec.withOverriddenContext(nativeQueryContext), MSQTaskQueryMakerUtils.maskSensitiveJsonKeys(plannerContext.getSql()), plannerContext.queryContextMap(), + SqlResults.Context.fromPlannerContext(plannerContext), sqlTypeNames, null ); diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java index 1bb247587af..e1348d19cb6 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java @@ -108,8 +108,12 @@ public class MultiStageQueryContext private static final String DEFAULT_DESTINATION = null; public static final String CTX_ROWS_PER_SEGMENT = "rowsPerSegment"; + static final int DEFAULT_ROWS_PER_SEGMENT = 3000000; public static final String CTX_ROWS_IN_MEMORY = "rowsInMemory"; + // Lower than the default to minimize the impact of per-row overheads that are not accounted for by + // OnheapIncrementalIndex. For example: overheads related to creating bitmaps during persist. + static final int DEFAULT_ROWS_IN_MEMORY = 100000; /** * Controls sort order within segments. Normally, this is the same as the overall order of the query (from the @@ -215,20 +219,17 @@ public class MultiStageQueryContext ); } - public static int getRowsPerSegment(final QueryContext queryContext, int defaultRowsPerSegment) + public static int getRowsPerSegment(final QueryContext queryContext) { return queryContext.getInt( CTX_ROWS_PER_SEGMENT, - defaultRowsPerSegment + DEFAULT_ROWS_PER_SEGMENT ); } - public static int getRowsInMemory(final QueryContext queryContext, int defaultRowsInMemory) + public static int getRowsInMemory(final QueryContext queryContext) { - return queryContext.getInt( - CTX_ROWS_IN_MEMORY, - defaultRowsInMemory - ); + return queryContext.getInt(CTX_ROWS_IN_MEMORY, DEFAULT_ROWS_IN_MEMORY); } public static List getSortOrder(final QueryContext queryContext) diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java index 31d8afd223a..3ee7eab9a40 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java @@ -141,7 +141,7 @@ public class MSQSelectTest extends MSQTestBase ) .setExpectedRowSignature(resultSignature) .setQueryContext(context) - .setExpectedResultRows(ImmutableList.of(new Object[]{2L})).verifyResults(); + .setExpectedResultRows(ImmutableList.of(new Object[]{2})).verifyResults(); } @Test @@ -186,7 +186,7 @@ public class MSQSelectTest extends MSQTestBase 0, 0, "shuffle" ) .setExpectedResultRows(ImmutableList.of( - new Object[]{1L, !useDefault ? "" : null}, + new Object[]{1L, ""}, new Object[]{1L, "10.1"}, new Object[]{1L, "2"}, new Object[]{1L, "1"}, @@ -306,7 +306,7 @@ public class MSQSelectTest extends MSQTestBase 0, 0, "shuffle" ) .setExpectedResultRows(ImmutableList.of( - new Object[]{1L, !useDefault ? "" : null}, + new Object[]{1L, ""}, new Object[]{1L, "10.1"}, new Object[]{1L, "2"}, new Object[]{1L, "1"}, @@ -525,7 +525,7 @@ public class MSQSelectTest extends MSQTestBase 0, 0, "shuffle" ) .setExpectedResultRows(ImmutableList.of( - new Object[]{1L, !useDefault ? "" : null}, + new Object[]{1L, ""}, new Object[]{1L, "10.1"}, new Object[]{1L, "2"}, new Object[]{1L, "1"}, @@ -752,7 +752,7 @@ public class MSQSelectTest extends MSQTestBase ); } else { expectedResults = ImmutableList.of( - new Object[]{null, 3.6666666666666665}, + new Object[]{"", 3.6666666666666665}, new Object[]{"a", 2.5}, new Object[]{"abc", 5.0} ); @@ -1316,31 +1316,51 @@ public class MSQSelectTest extends MSQTestBase @Test public void testScanWithMultiValueSelectQuery() { - RowSignature resultSignature = RowSignature.builder() - .add("dim3", ColumnType.STRING) - .build(); + RowSignature expectedScanSignature = RowSignature.builder() + .add("dim3", ColumnType.STRING) + .add("v0", ColumnType.STRING_ARRAY) + .build(); + + RowSignature expectedResultSignature = RowSignature.builder() + .add("dim3", ColumnType.STRING) + .add("dim3_array", ColumnType.STRING_ARRAY) + .build(); testSelectQuery() - .setSql("select dim3 from foo") + .setSql("select dim3, MV_TO_ARRAY(dim3) AS dim3_array from foo") .setExpectedMSQSpec(MSQSpec.builder() .query(newScanQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) .intervals(querySegmentSpec(Filtration.eternity())) - .columns("dim3") - .context(defaultScanQueryContext(context, resultSignature)) + .virtualColumns( + expressionVirtualColumn( + "v0", + "mv_to_array(\"dim3\")", + ColumnType.STRING_ARRAY + ) + ) + .columns("dim3", "v0") + .context(defaultScanQueryContext(context, expectedScanSignature)) .build()) - .columnMappings(ColumnMappings.identity(resultSignature)) + .columnMappings( + new ColumnMappings( + ImmutableList.of( + new ColumnMapping("dim3", "dim3"), + new ColumnMapping("v0", "dim3_array") + ) + ) + ) .tuningConfig(MSQTuningConfig.defaultConfig()) .build()) - .setExpectedRowSignature(resultSignature) + .setExpectedRowSignature(expectedResultSignature) .setQueryContext(context) .setExpectedResultRows(ImmutableList.of( - new Object[]{ImmutableList.of("a", "b")}, - new Object[]{ImmutableList.of("b", "c")}, - new Object[]{"d"}, - new Object[]{!useDefault ? "" : null}, - new Object[]{null}, - new Object[]{null} + new Object[]{"[\"a\",\"b\"]", ImmutableList.of("a", "b")}, + new Object[]{"[\"b\",\"c\"]", ImmutableList.of("b", "c")}, + new Object[]{"d", ImmutableList.of("d")}, + new Object[]{"", Collections.singletonList(useDefault ? null : "")}, + new Object[]{NullHandling.defaultStringValue(), Collections.singletonList(null)}, + new Object[]{NullHandling.defaultStringValue(), Collections.singletonList(null)} )).verifyResults(); } @@ -1404,7 +1424,7 @@ public class MSQSelectTest extends MSQTestBase .setExpectedRowSignature(resultSignature) .setExpectedResultRows( NullHandling.replaceWithDefault() - ? ImmutableList.of(new Object[]{null, 3L}, new Object[]{"a", 2L}) + ? ImmutableList.of(new Object[]{"", 3L}, new Object[]{"a", 2L}) : ImmutableList.of(new Object[]{null, 2L}, new Object[]{"a", 2L}) ) @@ -1452,9 +1472,7 @@ public class MSQSelectTest extends MSQTestBase .tuningConfig(MSQTuningConfig.defaultConfig()) .build()) .setExpectedRowSignature(rowSignature) - .setExpectedResultRows( - expectedMultiValueFooRowsGroup() - ) + .setExpectedResultRows(expectedMultiValueFooRowsGroup()) .verifyResults(); } @@ -1833,8 +1851,10 @@ public class MSQSelectTest extends MSQTestBase private List expectedMultiValueFooRowsGroup() { ArrayList expected = new ArrayList<>(); - expected.add(new Object[]{null, !useDefault ? 2L : 3L}); - if (!useDefault) { + if (useDefault) { + expected.add(new Object[]{"", 3L}); + } else { + expected.add(new Object[]{null, 2L}); expected.add(new Object[]{"", 1L}); } expected.addAll(ImmutableList.of( diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQControllerTaskTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQControllerTaskTest.java index b61d4f4a2eb..37f0f915e1a 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQControllerTaskTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQControllerTaskTest.java @@ -62,6 +62,7 @@ public class MSQControllerTaskTest null, null, null, + null, null); Assert.assertTrue(msqWorkerTask.getInputSourceResources().isEmpty()); } diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/error/MSQWarningsTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/error/MSQWarningsTest.java index 1cc3335dc81..7ea821209b6 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/error/MSQWarningsTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/error/MSQWarningsTest.java @@ -32,6 +32,7 @@ import org.apache.druid.msq.test.MSQTestBase; import org.apache.druid.msq.test.MSQTestFileUtils; import org.apache.druid.msq.util.MultiStageQueryContext; import org.apache.druid.query.Query; +import org.apache.druid.query.QueryContexts; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.dimension.DefaultDimensionSpec; @@ -162,12 +163,7 @@ public class MSQWarningsTest extends MSQTestBase + " '[{\"name\": \"timestamp\", \"type\": \"string\"}, {\"name\": \"page\", \"type\": \"string\"}, {\"name\": \"user\", \"type\": \"string\"}]'\n" + " )\n" + ") group by 1") - .setQueryContext( - ImmutableMap.builder() - .putAll(DEFAULT_MSQ_CONTEXT) - .putAll(userContext) - .build() - ) + .setQueryContext(QueryContexts.override(DEFAULT_MSQ_CONTEXT, userContext)) .setExpectedRowSignature(rowSignature) .setExpectedResultRows(ImmutableList.of(new Object[]{1566172800000L, 10L})) .setExpectedMSQSpec( @@ -250,12 +246,38 @@ public class MSQWarningsTest extends MSQTestBase + " '[{\"name\": \"timestamp\", \"type\": \"string\"}, {\"name\": \"page\", \"type\": \"string\"}, {\"name\": \"user\", \"type\": \"string\"}]'\n" + " )\n" + ") group by 1") - .setQueryContext( - ImmutableMap.builder() - .putAll(DEFAULT_MSQ_CONTEXT) - .putAll(userContext) - .build() - ) + .setQueryContext(QueryContexts.override(DEFAULT_MSQ_CONTEXT, userContext)) + .setExpectedRowSignature(rowSignature) + .setExpectedResultRows(ImmutableList.of(new Object[]{1566172800000L, 10L})) + .setExpectedMSQSpec( + MSQSpec.builder() + .query(defaultQuery.withOverriddenContext(userContext)) + .columnMappings(defaultColumnMappings) + .tuningConfig(MSQTuningConfig.defaultConfig()) + .build()) + .verifyResults(); + } + + @Test + public void testSuccessWhenParseExceptionsOnLimitOverridesMode() + { + final Map userContext = + ImmutableMap.of( + MultiStageQueryContext.CTX_MSQ_MODE, "strict", + MSQWarnings.CTX_MAX_PARSE_EXCEPTIONS_ALLOWED, 10 // Takes precedence over "strict" mode + ); + + testSelectQuery().setSql("SELECT\n" + + " floor(TIME_PARSE(\"timestamp\") to day) AS __time,\n" + + " count(*) as cnt\n" + + "FROM TABLE(\n" + + " EXTERN(\n" + + " '{ \"files\": [\"" + toRead.getAbsolutePath() + "\"],\"type\":\"local\"}',\n" + + " '{\"type\": \"json\"}',\n" + + " '[{\"name\": \"timestamp\", \"type\": \"string\"}, {\"name\": \"page\", \"type\": \"string\"}, {\"name\": \"user\", \"type\": \"string\"}]'\n" + + " )\n" + + ") group by 1") + .setQueryContext(QueryContexts.override(DEFAULT_MSQ_CONTEXT, userContext)) .setExpectedRowSignature(rowSignature) .setExpectedResultRows(ImmutableList.of(new Object[]{1566172800000L, 10L})) .setExpectedMSQSpec( @@ -270,7 +292,9 @@ public class MSQWarningsTest extends MSQTestBase @Test public void testSuccessInNonStrictMode() { - final Map userContext = ImmutableMap.of(MultiStageQueryContext.CTX_MSQ_MODE, "nonStrict"); + final Map userContext = + QueryContexts.override(DEFAULT_MSQ_CONTEXT, ImmutableMap.of(MultiStageQueryContext.CTX_MSQ_MODE, "nonStrict")); + userContext.remove(MSQWarnings.CTX_MAX_PARSE_EXCEPTIONS_ALLOWED); testSelectQuery().setSql("SELECT\n" + " floor(TIME_PARSE(\"timestamp\") to day) AS __time,\n" @@ -282,17 +306,20 @@ public class MSQWarningsTest extends MSQTestBase + " '[{\"name\": \"timestamp\", \"type\": \"string\"}, {\"name\": \"page\", \"type\": \"string\"}, {\"name\": \"user\", \"type\": \"string\"}]'\n" + " )\n" + ") group by 1") - .setQueryContext( - ImmutableMap.builder() - .putAll(DEFAULT_MSQ_CONTEXT) - .putAll(userContext) - .build() - ) + .setQueryContext(userContext) .setExpectedRowSignature(rowSignature) .setExpectedResultRows(ImmutableList.of(new Object[]{1566172800000L, 10L})) .setExpectedMSQSpec( MSQSpec.builder() - .query(defaultQuery.withOverriddenContext(userContext)) + .query( + defaultQuery.withOverriddenContext(userContext) + .withOverriddenContext( + ImmutableMap.of( + MSQWarnings.CTX_MAX_PARSE_EXCEPTIONS_ALLOWED, + -1 + ) + ) + ) .columnMappings(defaultColumnMappings) .tuningConfig(MSQTuningConfig.defaultConfig()) .build()) @@ -339,9 +366,7 @@ public class MSQWarningsTest extends MSQTestBase + " '[{\"name\": \"timestamp\", \"type\": \"string\"}, {\"name\": \"page\", \"type\": \"string\"}, {\"name\": \"user\", \"type\": \"string\"}]'\n" + " )\n" + ") group by 1 PARTITIONED by day ") - .setQueryContext(new ImmutableMap.Builder().putAll(DEFAULT_MSQ_CONTEXT) - .putAll(ROLLUP_CONTEXT_PARAMS) - .build()) + .setQueryContext(QueryContexts.override(DEFAULT_MSQ_CONTEXT, ROLLUP_CONTEXT_PARAMS)) .setExpectedRollUp(true) .setExpectedDataSource("foo1") .setExpectedRowSignature(rowSignature) @@ -391,12 +416,7 @@ public class MSQWarningsTest extends MSQTestBase + " '[{\"name\": \"timestamp\", \"type\": \"string\"}, {\"name\": \"page\", \"type\": \"string\"}, {\"name\": \"user\", \"type\": \"string\"}]'\n" + " )\n" + ") group by 1") - .setQueryContext( - ImmutableMap.builder() - .putAll(DEFAULT_MSQ_CONTEXT) - .putAll(userContext) - .build() - ) + .setQueryContext(QueryContexts.override(DEFAULT_MSQ_CONTEXT, userContext)) .setExpectedRowSignature(rowSignature) .setExpectedResultRows(ImmutableList.of(new Object[]{1566172800000L, 10L})) .setExpectedMSQSpec( diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/report/MSQTaskReportTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/report/MSQTaskReportTest.java index b4baaea45a7..aeca792024a 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/report/MSQTaskReportTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/report/MSQTaskReportTest.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.frame.key.ClusterBy; import org.apache.druid.frame.key.KeyColumn; import org.apache.druid.frame.key.KeyOrder; @@ -103,7 +104,7 @@ public class MSQTaskReportTest new CounterSnapshotsTree(), new MSQResultsReport( Collections.singletonList(new MSQResultsReport.ColumnAndType("s", ColumnType.STRING)), - ImmutableList.of("VARCHAR"), + ImmutableList.of(SqlTypeName.VARCHAR), Yielders.each(Sequences.simple(results)) ) ) diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/CalciteSelectQueryMSQTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/CalciteSelectQueryMSQTest.java index 1f45b6ca160..33d99a7c456 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/CalciteSelectQueryMSQTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/CalciteSelectQueryMSQTest.java @@ -172,7 +172,6 @@ public class CalciteSelectQueryMSQTest extends CalciteQueryTest @Override public void testArrayAggQueryOnComplexDatatypes() { - msqCompatible(); try { testQuery("SELECT ARRAY_AGG(unique_dim1) FROM druid.foo", ImmutableList.of(), ImmutableList.of()); Assert.fail("query execution should fail"); diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java index a74e51f1013..6b50d846736 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java @@ -90,6 +90,7 @@ import org.apache.druid.msq.indexing.error.InsertLockPreemptedFaultTest; import org.apache.druid.msq.indexing.error.MSQErrorReport; import org.apache.druid.msq.indexing.error.MSQFault; import org.apache.druid.msq.indexing.error.MSQFaultUtils; +import org.apache.druid.msq.indexing.error.MSQWarnings; import org.apache.druid.msq.indexing.error.TooManyAttemptsForWorker; import org.apache.druid.msq.indexing.report.MSQResultsReport; import org.apache.druid.msq.indexing.report.MSQTaskReport; @@ -223,6 +224,8 @@ public class MSQTestBase extends BaseCalciteQueryTest ImmutableMap.builder() .put(QueryContexts.CTX_SQL_QUERY_ID, "test-query") .put(QueryContexts.FINALIZE_KEY, true) + .put(QueryContexts.CTX_SQL_STRINGIFY_ARRAYS, false) + .put(MSQWarnings.CTX_MAX_PARSE_EXCEPTIONS_ALLOWED, 0) .build(); public static final Map DURABLE_STORAGE_MSQ_CONTEXT = diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/MultiStageQueryContextTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/MultiStageQueryContextTest.java index cb6b2fdb972..c118a40e89a 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/MultiStageQueryContextTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/MultiStageQueryContextTest.java @@ -156,27 +156,33 @@ public class MultiStageQueryContextTest @Test public void getRowsPerSegment_noParameterSetReturnsDefaultValue() { - Assert.assertEquals(1000, MultiStageQueryContext.getRowsPerSegment(QueryContext.empty(), 1000)); + Assert.assertEquals( + MultiStageQueryContext.DEFAULT_ROWS_PER_SEGMENT, + MultiStageQueryContext.getRowsPerSegment(QueryContext.empty()) + ); } @Test public void getRowsPerSegment_parameterSetReturnsCorrectValue() { Map propertyMap = ImmutableMap.of(CTX_ROWS_PER_SEGMENT, 10); - Assert.assertEquals(10, MultiStageQueryContext.getRowsPerSegment(QueryContext.of(propertyMap), 1000)); + Assert.assertEquals(10, MultiStageQueryContext.getRowsPerSegment(QueryContext.of(propertyMap))); } @Test public void getRowsInMemory_noParameterSetReturnsDefaultValue() { - Assert.assertEquals(1000, MultiStageQueryContext.getRowsInMemory(QueryContext.empty(), 1000)); + Assert.assertEquals( + MultiStageQueryContext.DEFAULT_ROWS_IN_MEMORY, + MultiStageQueryContext.getRowsInMemory(QueryContext.empty()) + ); } @Test public void getRowsInMemory_parameterSetReturnsCorrectValue() { Map propertyMap = ImmutableMap.of(CTX_ROWS_IN_MEMORY, 10); - Assert.assertEquals(10, MultiStageQueryContext.getRowsInMemory(QueryContext.of(propertyMap), 1000)); + Assert.assertEquals(10, MultiStageQueryContext.getRowsInMemory(QueryContext.of(propertyMap))); } @Test diff --git a/integration-tests-ex/cases/src/test/resources/multi-stage-query/wikipedia_msq_select_query1.json b/integration-tests-ex/cases/src/test/resources/multi-stage-query/wikipedia_msq_select_query1.json index 32c3d592d6e..151fb54aaff 100644 --- a/integration-tests-ex/cases/src/test/resources/multi-stage-query/wikipedia_msq_select_query1.json +++ b/integration-tests-ex/cases/src/test/resources/multi-stage-query/wikipedia_msq_select_query1.json @@ -4,7 +4,7 @@ "expectedResults": [ { "__time": 1377910953000, - "isRobot": null, + "isRobot": "", "added": 57, "delta": -143, "deleted": 200, @@ -12,7 +12,7 @@ }, { "__time": 1377919965000, - "isRobot": null, + "isRobot": "", "added": 459, "delta": 330, "deleted": 129, @@ -20,7 +20,7 @@ }, { "__time": 1377933081000, - "isRobot": null, + "isRobot": "", "added": 123, "delta": 111, "deleted": 12, diff --git a/integration-tests-ex/cases/src/test/resources/multi-stage-query/wikipedia_msq_select_query_ha.json b/integration-tests-ex/cases/src/test/resources/multi-stage-query/wikipedia_msq_select_query_ha.json index 992eda01a26..58c38250722 100644 --- a/integration-tests-ex/cases/src/test/resources/multi-stage-query/wikipedia_msq_select_query_ha.json +++ b/integration-tests-ex/cases/src/test/resources/multi-stage-query/wikipedia_msq_select_query_ha.json @@ -4,7 +4,7 @@ "expectedResults": [ { "__time": 1377910953000, - "isRobot": null, + "isRobot": "", "added": 57, "delta": -143, "deleted": 200, @@ -12,7 +12,7 @@ }, { "__time": 1377910953000, - "isRobot": null, + "isRobot": "", "added": 57, "delta": -143, "deleted": 200, @@ -20,7 +20,7 @@ }, { "__time": 1377919965000, - "isRobot": null, + "isRobot": "", "added": 459, "delta": 330, "deleted": 129, @@ -28,7 +28,7 @@ }, { "__time": 1377919965000, - "isRobot": null, + "isRobot": "", "added": 459, "delta": 330, "deleted": 129, @@ -36,7 +36,7 @@ }, { "__time": 1377933081000, - "isRobot": null, + "isRobot": "", "added": 123, "delta": 111, "deleted": 12, @@ -44,7 +44,7 @@ }, { "__time": 1377933081000, - "isRobot": null, + "isRobot": "", "added": 123, "delta": 111, "deleted": 12, diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeQueryMaker.java b/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeQueryMaker.java index c57611a1218..e2b30591901 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeQueryMaker.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeQueryMaker.java @@ -19,26 +19,20 @@ package org.apache.druid.sql.calcite.run; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Iterables; -import com.google.common.primitives.Ints; import it.unimi.dsi.fastutil.objects.Object2IntMap; import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.runtime.Hook; -import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.util.NlsString; import org.apache.calcite.util.Pair; -import org.apache.druid.common.config.NullHandling; -import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.UOE; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.Sequences; -import org.apache.druid.math.expr.Evals; import org.apache.druid.query.InlineDataSource; import org.apache.druid.query.Query; import org.apache.druid.query.QueryToolChest; @@ -47,27 +41,19 @@ import org.apache.druid.query.filter.DimFilter; import org.apache.druid.query.filter.OrDimFilter; import org.apache.druid.query.spec.QuerySegmentSpec; import org.apache.druid.query.timeseries.TimeseriesQuery; -import org.apache.druid.segment.DimensionHandlerUtils; import org.apache.druid.segment.column.ColumnHolder; -import org.apache.druid.segment.data.ComparableList; -import org.apache.druid.segment.data.ComparableStringArray; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryResponse; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.AuthenticationResult; -import org.apache.druid.sql.calcite.planner.Calcites; import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.rel.CannotBuildQueryException; import org.apache.druid.sql.calcite.rel.DruidQuery; -import org.joda.time.DateTime; import org.joda.time.Interval; -import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; @@ -150,11 +136,11 @@ public class NativeQueryMaker implements QueryMaker rowOrder = druidQuery.getOutputRowSignature().getColumnNames(); } - final List columnTypes = + final List columnTypes = druidQuery.getOutputRowType() .getFieldList() .stream() - .map(f -> f.getType().getSqlTypeName()) + .map(RelDataTypeField::getType) .collect(Collectors.toList()); return execute( @@ -173,7 +159,11 @@ public class NativeQueryMaker implements QueryMaker } @SuppressWarnings("unchecked") - private QueryResponse execute(Query query, final List newFields, final List newTypes) + private QueryResponse execute( + Query query, // Not final: may be reassigned with query ID added + final List newFields, + final List newTypes + ) { Hook.QUERY_PLAN.run(query); @@ -211,7 +201,7 @@ public class NativeQueryMaker implements QueryMaker final QueryToolChest> toolChest, final Query query, final List newFields, - final List newTypes + final List newTypes ) { final List originalFields = toolChest.resultArraySignature(query).getColumnNames(); @@ -240,13 +230,19 @@ public class NativeQueryMaker implements QueryMaker } final Sequence sequence = toolChest.resultsAsArrays(query, results.getResults()); + final SqlResults.Context sqlResultsContext = SqlResults.Context.fromPlannerContext(plannerContext); return new QueryResponse<>( Sequences.map( sequence, array -> { final Object[] newArray = new Object[mapping.length]; for (int i = 0; i < mapping.length; i++) { - newArray[i] = coerce(array[mapping[i]], newTypes.get(i)); + newArray[i] = SqlResults.coerce( + jsonMapper, + sqlResultsContext, + array[mapping[i]], + newTypes.get(i).getSqlTypeName() + ); } return newArray; } @@ -255,175 +251,6 @@ public class NativeQueryMaker implements QueryMaker ); } - private Object coerce(final Object value, final SqlTypeName sqlType) - { - final Object coercedValue; - - if (SqlTypeName.CHAR_TYPES.contains(sqlType)) { - if (value == null || value instanceof String) { - coercedValue = NullHandling.nullToEmptyIfNeeded((String) value); - } else if (value instanceof NlsString) { - coercedValue = ((NlsString) value).getValue(); - } else if (value instanceof Number) { - coercedValue = String.valueOf(value); - } else if (value instanceof Boolean) { - coercedValue = String.valueOf(value); - } else if (value instanceof Collection) { - // Iterate through the collection, coercing each value. Useful for handling selects of multi-value dimensions. - final List valueStrings = ((Collection) value).stream() - .map(v -> (String) coerce(v, sqlType)) - .collect(Collectors.toList()); - - try { - coercedValue = jsonMapper.writeValueAsString(valueStrings); - } - catch (IOException e) { - throw new RuntimeException(e); - } - } else { - throw new ISE("Cannot coerce [%s] to %s", value.getClass().getName(), sqlType); - } - } else if (value == null) { - coercedValue = null; - } else if (sqlType == SqlTypeName.DATE) { - return Calcites.jodaToCalciteDate(coerceDateTime(value, sqlType), plannerContext.getTimeZone()); - } else if (sqlType == SqlTypeName.TIMESTAMP) { - return Calcites.jodaToCalciteTimestamp(coerceDateTime(value, sqlType), plannerContext.getTimeZone()); - } else if (sqlType == SqlTypeName.BOOLEAN) { - if (value instanceof String) { - coercedValue = Evals.asBoolean(((String) value)); - } else if (value instanceof Number) { - coercedValue = Evals.asBoolean(((Number) value).longValue()); - } else { - throw new ISE("Cannot coerce [%s] to %s", value.getClass().getName(), sqlType); - } - } else if (sqlType == SqlTypeName.INTEGER) { - if (value instanceof String) { - coercedValue = Ints.tryParse((String) value); - } else if (value instanceof Number) { - coercedValue = ((Number) value).intValue(); - } else { - throw new ISE("Cannot coerce [%s] to %s", value.getClass().getName(), sqlType); - } - } else if (sqlType == SqlTypeName.BIGINT) { - try { - coercedValue = DimensionHandlerUtils.convertObjectToLong(value); - } - catch (Exception e) { - throw new ISE("Cannot coerce [%s] to %s", value.getClass().getName(), sqlType); - } - } else if (sqlType == SqlTypeName.FLOAT) { - try { - coercedValue = DimensionHandlerUtils.convertObjectToFloat(value); - } - catch (Exception e) { - throw new ISE("Cannot coerce [%s] to %s", value.getClass().getName(), sqlType); - } - } else if (SqlTypeName.FRACTIONAL_TYPES.contains(sqlType)) { - try { - coercedValue = DimensionHandlerUtils.convertObjectToDouble(value); - } - catch (Exception e) { - throw new ISE("Cannot coerce [%s] to %s", value.getClass().getName(), sqlType); - } - } else if (sqlType == SqlTypeName.OTHER) { - // Complex type, try to serialize if we should, else print class name - if (plannerContext.getPlannerConfig().shouldSerializeComplexValues()) { - try { - coercedValue = jsonMapper.writeValueAsString(value); - } - catch (JsonProcessingException jex) { - throw new ISE(jex, "Cannot coerce [%s] to %s", value.getClass().getName(), sqlType); - } - } else { - coercedValue = value.getClass().getName(); - } - } else if (sqlType == SqlTypeName.ARRAY) { - if (plannerContext.isStringifyArrays()) { - if (value instanceof String) { - coercedValue = NullHandling.nullToEmptyIfNeeded((String) value); - } else if (value instanceof NlsString) { - coercedValue = ((NlsString) value).getValue(); - } else { - try { - coercedValue = jsonMapper.writeValueAsString(value); - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - } else { - // the protobuf jdbc handler prefers lists (it actually can't handle java arrays as sql arrays, only java lists) - // the json handler could handle this just fine, but it handles lists as sql arrays as well so just convert - // here if needed - coercedValue = maybeCoerceArrayToList(value, true); - if (coercedValue == null) { - throw new ISE("Cannot coerce [%s] to %s", value.getClass().getName(), sqlType); - } - } - } else { - throw new ISE("Cannot coerce [%s] to %s", value.getClass().getName(), sqlType); - } - - return coercedValue; - } - - - @VisibleForTesting - static Object maybeCoerceArrayToList(Object value, boolean mustCoerce) - { - if (value instanceof List) { - return value; - } else if (value instanceof String[]) { - return Arrays.asList((String[]) value); - } else if (value instanceof Long[]) { - return Arrays.asList((Long[]) value); - } else if (value instanceof Double[]) { - return Arrays.asList((Double[]) value); - } else if (value instanceof Object[]) { - final Object[] array = (Object[]) value; - final ArrayList lst = new ArrayList<>(array.length); - for (Object o : array) { - lst.add(maybeCoerceArrayToList(o, false)); - } - return lst; - } else if (value instanceof long[]) { - return Arrays.stream((long[]) value).boxed().collect(Collectors.toList()); - } else if (value instanceof double[]) { - return Arrays.stream((double[]) value).boxed().collect(Collectors.toList()); - } else if (value instanceof float[]) { - final float[] array = (float[]) value; - final ArrayList lst = new ArrayList<>(array.length); - for (float f : array) { - lst.add(f); - } - return lst; - } else if (value instanceof ComparableStringArray) { - return Arrays.asList(((ComparableStringArray) value).getDelegate()); - } else if (value instanceof ComparableList) { - return ((ComparableList) value).getDelegate(); - } else if (mustCoerce) { - return null; - } - return value; - } - - private static DateTime coerceDateTime(Object value, SqlTypeName sqlType) - { - final DateTime dateTime; - - if (value instanceof Number) { - dateTime = DateTimes.utc(((Number) value).longValue()); - } else if (value instanceof String) { - dateTime = DateTimes.utc(Long.parseLong((String) value)); - } else if (value instanceof DateTime) { - dateTime = (DateTime) value; - } else { - throw new ISE("Cannot coerce[%s] to %s", value.getClass().getName(), sqlType); - } - return dateTime; - } - private static List mapColumnList(final List in, final List> fieldMapping) { final List out = new ArrayList<>(fieldMapping.size()); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/run/SqlResults.java b/sql/src/main/java/org/apache/druid/sql/calcite/run/SqlResults.java new file mode 100644 index 00000000000..d48eaa8742a --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/run/SqlResults.java @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.run; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.primitives.Ints; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.util.NlsString; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.math.expr.Evals; +import org.apache.druid.segment.DimensionHandlerUtils; +import org.apache.druid.segment.data.ComparableList; +import org.apache.druid.segment.data.ComparableStringArray; +import org.apache.druid.sql.calcite.planner.Calcites; +import org.apache.druid.sql.calcite.planner.PlannerContext; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Holder for the utility method {@link #coerce(ObjectMapper, Context, Object, SqlTypeName)}. + */ +public class SqlResults +{ + public static Object coerce( + final ObjectMapper jsonMapper, + final Context context, + final Object value, + final SqlTypeName sqlTypeName + ) + { + final Object coercedValue; + + if (SqlTypeName.CHAR_TYPES.contains(sqlTypeName)) { + if (value == null || value instanceof String) { + coercedValue = NullHandling.nullToEmptyIfNeeded((String) value); + } else if (value instanceof NlsString) { + coercedValue = ((NlsString) value).getValue(); + } else if (value instanceof Number) { + coercedValue = String.valueOf(value); + } else if (value instanceof Boolean) { + coercedValue = String.valueOf(value); + } else if (value instanceof Collection) { + // Iterate through the collection, coercing each value. Useful for handling selects of multi-value dimensions. + final List valueStrings = + ((Collection) value).stream() + .map(v -> (String) coerce(jsonMapper, context, v, sqlTypeName)) + .collect(Collectors.toList()); + + try { + coercedValue = jsonMapper.writeValueAsString(valueStrings); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } else { + throw new ISE("Cannot coerce [%s] to %s", value.getClass().getName(), sqlTypeName); + } + } else if (value == null) { + coercedValue = null; + } else if (sqlTypeName == SqlTypeName.DATE) { + return Calcites.jodaToCalciteDate(coerceDateTime(value, sqlTypeName), context.getTimeZone()); + } else if (sqlTypeName == SqlTypeName.TIMESTAMP) { + return Calcites.jodaToCalciteTimestamp(coerceDateTime(value, sqlTypeName), context.getTimeZone()); + } else if (sqlTypeName == SqlTypeName.BOOLEAN) { + if (value instanceof String) { + coercedValue = Evals.asBoolean(((String) value)); + } else if (value instanceof Number) { + coercedValue = Evals.asBoolean(((Number) value).longValue()); + } else { + throw new ISE("Cannot coerce [%s] to %s", value.getClass().getName(), sqlTypeName); + } + } else if (sqlTypeName == SqlTypeName.INTEGER) { + if (value instanceof String) { + coercedValue = Ints.tryParse((String) value); + } else if (value instanceof Number) { + coercedValue = ((Number) value).intValue(); + } else { + throw new ISE("Cannot coerce [%s] to %s", value.getClass().getName(), sqlTypeName); + } + } else if (sqlTypeName == SqlTypeName.BIGINT) { + try { + coercedValue = DimensionHandlerUtils.convertObjectToLong(value); + } + catch (Exception e) { + throw new ISE("Cannot coerce [%s] to %s", value.getClass().getName(), sqlTypeName); + } + } else if (sqlTypeName == SqlTypeName.FLOAT) { + try { + coercedValue = DimensionHandlerUtils.convertObjectToFloat(value); + } + catch (Exception e) { + throw new ISE("Cannot coerce [%s] to %s", value.getClass().getName(), sqlTypeName); + } + } else if (SqlTypeName.FRACTIONAL_TYPES.contains(sqlTypeName)) { + try { + coercedValue = DimensionHandlerUtils.convertObjectToDouble(value); + } + catch (Exception e) { + throw new ISE("Cannot coerce [%s] to %s", value.getClass().getName(), sqlTypeName); + } + } else if (sqlTypeName == SqlTypeName.OTHER) { + // Complex type, try to serialize if we should, else print class name + if (context.isSerializeComplexValues()) { + try { + coercedValue = jsonMapper.writeValueAsString(value); + } + catch (JsonProcessingException jex) { + throw new ISE(jex, "Cannot coerce [%s] to %s", value.getClass().getName(), sqlTypeName); + } + } else { + coercedValue = value.getClass().getName(); + } + } else if (sqlTypeName == SqlTypeName.ARRAY) { + if (context.isStringifyArrays()) { + if (value instanceof String) { + coercedValue = NullHandling.nullToEmptyIfNeeded((String) value); + } else if (value instanceof NlsString) { + coercedValue = ((NlsString) value).getValue(); + } else { + try { + coercedValue = jsonMapper.writeValueAsString(value); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + } else { + // the protobuf jdbc handler prefers lists (it actually can't handle java arrays as sql arrays, only java lists) + // the json handler could handle this just fine, but it handles lists as sql arrays as well so just convert + // here if needed + coercedValue = maybeCoerceArrayToList(value, true); + if (coercedValue == null) { + throw new ISE("Cannot coerce [%s] to %s", value.getClass().getName(), sqlTypeName); + } + } + } else { + throw new ISE("Cannot coerce [%s] to %s", value.getClass().getName(), sqlTypeName); + } + + return coercedValue; + } + + + @VisibleForTesting + static Object maybeCoerceArrayToList(Object value, boolean mustCoerce) + { + if (value instanceof List) { + return value; + } else if (value instanceof String[]) { + return Arrays.asList((String[]) value); + } else if (value instanceof Long[]) { + return Arrays.asList((Long[]) value); + } else if (value instanceof Double[]) { + return Arrays.asList((Double[]) value); + } else if (value instanceof Object[]) { + final Object[] array = (Object[]) value; + final ArrayList lst = new ArrayList<>(array.length); + for (Object o : array) { + lst.add(maybeCoerceArrayToList(o, false)); + } + return lst; + } else if (value instanceof long[]) { + return Arrays.stream((long[]) value).boxed().collect(Collectors.toList()); + } else if (value instanceof double[]) { + return Arrays.stream((double[]) value).boxed().collect(Collectors.toList()); + } else if (value instanceof float[]) { + final float[] array = (float[]) value; + final ArrayList lst = new ArrayList<>(array.length); + for (float f : array) { + lst.add(f); + } + return lst; + } else if (value instanceof ComparableStringArray) { + return Arrays.asList(((ComparableStringArray) value).getDelegate()); + } else if (value instanceof ComparableList) { + return ((ComparableList) value).getDelegate(); + } else if (mustCoerce) { + return null; + } + return value; + } + + private static DateTime coerceDateTime(Object value, SqlTypeName sqlType) + { + final DateTime dateTime; + + if (value instanceof Number) { + dateTime = DateTimes.utc(((Number) value).longValue()); + } else if (value instanceof String) { + dateTime = DateTimes.utc(Long.parseLong((String) value)); + } else if (value instanceof DateTime) { + dateTime = (DateTime) value; + } else { + throw new ISE("Cannot coerce[%s] to %s", value.getClass().getName(), sqlType); + } + return dateTime; + } + + /** + * Context for {@link #coerce(ObjectMapper, Context, Object, SqlTypeName)} + */ + public static class Context + { + private final DateTimeZone timeZone; + private final boolean serializeComplexValues; + private final boolean stringifyArrays; + + @JsonCreator + public Context( + @JsonProperty("timeZone") final DateTimeZone timeZone, + @JsonProperty("serializeComplexValues") final boolean serializeComplexValues, + @JsonProperty("stringifyArrays") final boolean stringifyArrays + ) + { + this.timeZone = timeZone; + this.serializeComplexValues = serializeComplexValues; + this.stringifyArrays = stringifyArrays; + } + + public static Context fromPlannerContext(final PlannerContext plannerContext) + { + return new Context( + plannerContext.getTimeZone(), + plannerContext.getPlannerConfig().shouldSerializeComplexValues(), + plannerContext.isStringifyArrays() + ); + } + + @JsonProperty + public DateTimeZone getTimeZone() + { + return timeZone; + } + + @JsonProperty + public boolean isSerializeComplexValues() + { + return serializeComplexValues; + } + + @JsonProperty + public boolean isStringifyArrays() + { + return stringifyArrays; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Context context = (Context) o; + return serializeComplexValues == context.serializeComplexValues + && stringifyArrays == context.stringifyArrays + && Objects.equals(timeZone, context.timeZone); + } + + @Override + public int hashCode() + { + return Objects.hash(timeZone, serializeComplexValues, stringifyArrays); + } + + @Override + public String toString() + { + return "Context{" + + "timeZone=" + timeZone + + ", serializeComplexValues=" + serializeComplexValues + + ", stringifyArrays=" + stringifyArrays + + '}'; + } + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index 43eb443045a..0bc2dc3ffc4 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -267,7 +267,7 @@ public class BaseCalciteQueryTest extends CalciteTestBase public boolean cannotVectorize = false; public boolean skipVectorize = false; - public boolean msqCompatible = false; + public boolean msqCompatible = true; public QueryLogHook queryLogHook; @@ -1010,9 +1010,9 @@ public class BaseCalciteQueryTest extends CalciteTestBase skipVectorize = true; } - protected void msqCompatible() + protected void notMsqCompatible() { - msqCompatible = true; + msqCompatible = false; } protected static boolean isRewriteJoinToFilter(final Map queryContext) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index cc728abb349..9d5b0e2c52b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -144,7 +144,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByWithPostAggregatorReferencingTimeFloorColumnOnTimeseries() { - msqCompatible(); cannotVectorize(); testQuery( @@ -191,6 +190,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testInformationSchemaSchemata() { + notMsqCompatible(); testQuery( "SELECT DISTINCT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA", ImmutableList.of(), @@ -207,6 +207,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testInformationSchemaTables() { + notMsqCompatible(); testQuery( "SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, IS_JOINABLE, IS_BROADCAST\n" + "FROM INFORMATION_SCHEMA.TABLES\n" @@ -283,6 +284,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testInformationSchemaColumnsOnTable() { + notMsqCompatible(); testQuery( "SELECT COLUMN_NAME, DATA_TYPE, IS_NULLABLE\n" + "FROM INFORMATION_SCHEMA.COLUMNS\n" @@ -304,6 +306,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testInformationSchemaColumnsOnForbiddenTable() { + notMsqCompatible(); testQuery( "SELECT COLUMN_NAME, DATA_TYPE, IS_NULLABLE\n" + "FROM INFORMATION_SCHEMA.COLUMNS\n" @@ -334,6 +337,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testInformationSchemaColumnsOnView() { + notMsqCompatible(); testQuery( "SELECT COLUMN_NAME, DATA_TYPE, IS_NULLABLE\n" + "FROM INFORMATION_SCHEMA.COLUMNS\n" @@ -348,6 +352,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testInformationSchemaColumnsOnAnotherView() { + notMsqCompatible(); testQuery( "SELECT COLUMN_NAME, DATA_TYPE, IS_NULLABLE\n" + "FROM INFORMATION_SCHEMA.COLUMNS\n" @@ -364,6 +369,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCannotInsertWithNativeEngine() { + notMsqCompatible(); final SqlPlanningException e = Assert.assertThrows( SqlPlanningException.class, () -> testQuery( @@ -384,6 +390,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCannotReplaceWithNativeEngine() { + notMsqCompatible(); final SqlPlanningException e = Assert.assertThrows( SqlPlanningException.class, () -> testQuery( @@ -404,6 +411,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testAggregatorsOnInformationSchemaColumns() { + notMsqCompatible(); // Not including COUNT DISTINCT, since it isn't supported by BindableAggregate, and so it can't work. testQuery( "SELECT\n" @@ -424,6 +432,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTopNLimitWrapping() { + notMsqCompatible(); List expected; if (NullHandling.replaceWithDefault()) { expected = ImmutableList.of( @@ -628,6 +637,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testEarliestAggregators() { + notMsqCompatible(); // Cannot vectorize EARLIEST aggregator. skipVectorize(); @@ -676,6 +686,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testLatestVectorAggregators() { + notMsqCompatible(); testQuery( "SELECT " + "LATEST(cnt), LATEST(cnt + 1), LATEST(m1), LATEST(m1+1) " @@ -709,6 +720,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testLatestAggregators() { + notMsqCompatible(); testQuery( "SELECT " @@ -755,7 +767,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testEarliestByInvalidTimestamp() { - msqCompatible(); expectedException.expect(SqlPlanningException.class); expectedException.expectMessage("Cannot apply 'EARLIEST_BY' to arguments of type 'EARLIEST_BY(, )"); @@ -769,7 +780,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testLatestByInvalidTimestamp() { - msqCompatible(); expectedException.expect(SqlPlanningException.class); expectedException.expectMessage("Cannot apply 'LATEST_BY' to arguments of type 'LATEST_BY(, )"); @@ -827,7 +837,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testAnyAggregatorsOnHeapNumericNulls() { - msqCompatible(); testQuery( "SELECT ANY_VALUE(l1), ANY_VALUE(d1), ANY_VALUE(f1) FROM druid.numfoo", ImmutableList.of( @@ -856,7 +865,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testAnyAggregatorsOffHeapNumericNulls() { - msqCompatible(); testQuery( "SELECT ANY_VALUE(l1), ANY_VALUE(d1), ANY_VALUE(f1) FROM druid.numfoo GROUP BY dim2", ImmutableList.of( @@ -895,6 +903,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testPrimitiveLatestInSubquery() { + notMsqCompatible(); testQuery( "SELECT SUM(val1), SUM(val2), SUM(val3) FROM (SELECT dim2, LATEST(m1) AS val1, LATEST(cnt) AS val2, LATEST(m2) AS val3 FROM foo GROUP BY dim2)", ImmutableList.of( @@ -942,6 +951,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testPrimitiveLatestInSubqueryGroupBy() { + notMsqCompatible(); testQuery( "SELECT dim2, LATEST(m1) AS val1 FROM foo GROUP BY dim2", ImmutableList.of( @@ -1009,7 +1019,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testStringLatestGroupByWithAlwaysFalseCondition() { - msqCompatible(); testQuery( "SELECT LATEST(dim4, 10),dim2 FROM numfoo WHERE (dim1 = 'something' AND dim1 IN( 'something else') ) GROUP BY dim2", ImmutableList.of( @@ -1035,7 +1044,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testStringLatestByGroupByWithAlwaysFalseCondition() { - msqCompatible(); testQuery( "SELECT LATEST_BY(dim4, __time, 10),dim2 FROM numfoo WHERE (dim1 = 'something' AND dim1 IN( 'something else') ) GROUP BY dim2", ImmutableList.of( @@ -1062,6 +1070,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testPrimitiveEarliestInSubquery() { + notMsqCompatible(); // Cannot vectorize EARLIEST aggregator. skipVectorize(); @@ -1113,7 +1122,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testStringLatestInSubquery() { - msqCompatible(); testQuery( "SELECT SUM(val) FROM (SELECT dim2, LATEST(dim1, 10) AS val FROM foo GROUP BY dim2)", ImmutableList.of( @@ -1162,7 +1170,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testStringEarliestInSubquery() { - msqCompatible(); // Cannot vectorize EARLIEST aggregator. skipVectorize(); @@ -1229,7 +1236,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest // null -> [2,3,6] | [2,3,6] // abc -> [5] | [5] // So the acceptable response can be any combination of these values - msqCompatible(); testQuery( "SELECT SUM(val1), SUM(val2), SUM(val3) FROM (SELECT dim2, ANY_VALUE(m1) AS val1, ANY_VALUE(cnt) AS val2, ANY_VALUE(m2) AS val3 FROM foo GROUP BY dim2)", ImmutableList.of( @@ -1278,7 +1284,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testStringAnyInSubquery() { - msqCompatible(); testQuery( "SELECT SUM(val) FROM (SELECT dim2, ANY_VALUE(dim1, 10) AS val FROM foo GROUP BY dim2)", ImmutableList.of( @@ -1325,6 +1330,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testEarliestAggregatorsNumericNulls() { + notMsqCompatible(); // Cannot vectorize EARLIEST aggregator. skipVectorize(); @@ -1354,6 +1360,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testLatestAggregatorsNumericNull() { + notMsqCompatible(); testQuery( "SELECT LATEST(l1), LATEST(d1), LATEST(f1) FROM druid.numfoo", ImmutableList.of( @@ -1384,6 +1391,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFirstLatestAggregatorsSkipNulls() { + notMsqCompatible(); // Cannot vectorize EARLIEST aggregator. skipVectorize(); @@ -1457,7 +1465,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testAnyAggregatorsSkipNullsWithFilter() { - msqCompatible(); final DimFilter filter; if (useDefault) { filter = not(selector("dim1", null, null)); @@ -1500,6 +1507,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testOrderByEarliestFloat() { + notMsqCompatible(); // Cannot vectorize EARLIEST aggregator. skipVectorize(); List expected; @@ -1547,6 +1555,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testOrderByEarliestDouble() { + notMsqCompatible(); // Cannot vectorize EARLIEST aggregator. skipVectorize(); List expected; @@ -1594,6 +1603,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testOrderByEarliestLong() { + notMsqCompatible(); // Cannot vectorize EARLIEST aggregator. skipVectorize(); List expected; @@ -1641,6 +1651,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testOrderByLatestFloat() { + notMsqCompatible(); List expected; if (NullHandling.replaceWithDefault()) { expected = ImmutableList.of( @@ -1687,6 +1698,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testOrderByLatestDouble() { + notMsqCompatible(); List expected; if (NullHandling.replaceWithDefault()) { expected = ImmutableList.of( @@ -1732,6 +1744,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testOrderByLatestLong() { + notMsqCompatible(); List expected; if (NullHandling.replaceWithDefault()) { expected = ImmutableList.of( @@ -1788,6 +1801,9 @@ public class CalciteQueryTest extends BaseCalciteQueryTest new Object[]{"", 1.0f} ); } else { + // Disabled test in MSQ with SQL-compatible mode till https://github.com/apache/druid/issues/13951 is resolved + notMsqCompatible(); + expected = ImmutableList.of( new Object[]{"2", 0.0f}, new Object[]{"10.1", 0.1f}, @@ -1836,6 +1852,9 @@ public class CalciteQueryTest extends BaseCalciteQueryTest new Object[]{"10.1", 1.7} ); } else { + // Disabled test in MSQ with SQL-compatible mode till https://github.com/apache/druid/issues/13951 is resolved + notMsqCompatible(); + expected = ImmutableList.of( new Object[]{"2", 0.0}, new Object[]{"", 1.0}, @@ -1883,6 +1902,9 @@ public class CalciteQueryTest extends BaseCalciteQueryTest new Object[]{"10.1", 325323L} ); } else { + // Disabled test in MSQ with SQL-compatible mode till https://github.com/apache/druid/issues/13951 is resolved + notMsqCompatible(); + expected = ImmutableList.of( new Object[]{"2", 0L}, new Object[]{"", 7L}, @@ -1919,7 +1941,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByLong() { - msqCompatible(); testQuery( "SELECT cnt, COUNT(*) FROM druid.foo GROUP BY cnt", ImmutableList.of( @@ -1941,7 +1962,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByOrdinal() { - msqCompatible(); testQuery( "SELECT cnt, COUNT(*) FROM druid.foo GROUP BY 1", ImmutableList.of( @@ -1964,6 +1984,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Ignore("Disabled since GROUP BY alias can confuse the validator; see DruidConformance::isGroupByAlias") public void testGroupByAndOrderByAlias() { + notMsqCompatible(); testQuery( "SELECT cnt AS theCnt, COUNT(*) FROM druid.foo GROUP BY theCnt ORDER BY theCnt ASC", ImmutableList.of( @@ -1997,7 +2018,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByExpressionAliasedAsOriginalColumnName() { - msqCompatible(); testQuery( "SELECT\n" + "FLOOR(__time TO MONTH) AS __time,\n" @@ -2023,7 +2043,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByAndOrderByOrdinalOfAlias() { - msqCompatible(); testQuery( "SELECT cnt as theCnt, COUNT(*) FROM druid.foo GROUP BY 1 ORDER BY 1 ASC", ImmutableList.of( @@ -2046,7 +2065,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByFloat() { - msqCompatible(); testQuery( "SELECT m1, COUNT(*) FROM druid.foo GROUP BY m1", ImmutableList.of( @@ -2073,7 +2091,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByDouble() { - msqCompatible(); testQuery( "SELECT m2, COUNT(*) FROM druid.foo GROUP BY m2", ImmutableList.of( @@ -2100,7 +2117,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilterOnFloat() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo WHERE m1 = 1.0", ImmutableList.of( @@ -2122,7 +2138,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilterOnDouble() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo WHERE m2 = 1.0", ImmutableList.of( @@ -2144,7 +2159,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testHavingOnGrandTotal() { - msqCompatible(); testQuery( "SELECT SUM(m1) AS m1_sum FROM foo HAVING m1_sum = 21", ImmutableList.of( @@ -2166,7 +2180,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testHavingOnDoubleSum() { - msqCompatible(); testQuery( "SELECT dim1, SUM(m1) AS m1_sum FROM druid.foo GROUP BY dim1 HAVING SUM(m1) > 1", ImmutableList.of( @@ -2324,6 +2337,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testExactCountDistinctWithFilter() { + notMsqCompatible(); final String sqlQuery = "SELECT COUNT(DISTINCT foo.dim1) FILTER(WHERE foo.cnt = 1), SUM(foo.cnt) FROM druid.foo"; // When useApproximateCountDistinct=true and useGroupingSetForExactDistinct=false, planning fails due // to a bug in the Calcite's rule (AggregateExpandDistinctAggregatesRule) @@ -2489,7 +2503,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testHavingOnRatio() { - msqCompatible(); // Test for https://github.com/apache/druid/issues/4264 testQuery( @@ -2684,6 +2697,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUnionAllQueries() { + notMsqCompatible(); testQuery( "SELECT COUNT(*) FROM foo UNION ALL SELECT SUM(cnt) FROM foo UNION ALL SELECT COUNT(*) FROM foo", ImmutableList.of( @@ -2716,6 +2730,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUnionAllQueriesWithLimit() { + notMsqCompatible(); testQuery( "SELECT * FROM (" + "SELECT COUNT(*) FROM foo UNION ALL SELECT SUM(cnt) FROM foo UNION ALL SELECT COUNT(*) FROM foo" @@ -2743,6 +2758,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUnionAllDifferentTablesWithMapping() { + notMsqCompatible(); testQuery( "SELECT\n" + "dim1, dim2, SUM(m1), COUNT(*)\n" @@ -2785,6 +2801,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testJoinUnionAllDifferentTablesWithMapping() { + notMsqCompatible(); testQuery( "SELECT\n" + "dim1, dim2, SUM(m1), COUNT(*)\n" @@ -2827,7 +2844,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUnionAllTablesColumnCountMismatch() { - msqCompatible(); try { testQuery( "SELECT\n" @@ -2852,6 +2868,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUnionAllTablesColumnTypeMismatchFloatLong() { + notMsqCompatible(); // "m1" has a different type in foo and foo2 (float vs long), but this query is OK anyway because they can both // be implicitly cast to double. @@ -2898,7 +2915,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUnionAllTablesColumnTypeMismatchStringLong() { - msqCompatible(); // "dim3" has a different type in foo and foo2 (string vs long), which requires a casting subquery, so this // query cannot be planned. @@ -2916,8 +2932,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest public void testUnionAllTablesWhenMappingIsRequired() { // Cannot plan this UNION ALL operation, because the column swap would require generating a subquery. - - msqCompatible(); assertQueryIsUnplannable( "SELECT\n" + "c, COUNT(*)\n" @@ -2933,8 +2947,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest public void testUnionIsUnplannable() { // Cannot plan this UNION operation - - msqCompatible(); assertQueryIsUnplannable( "SELECT dim2, dim1, m1 FROM foo2 UNION SELECT dim1, dim2, m1 FROM foo", "Possible error: SQL requires 'UNION' but only 'UNION ALL' is supported." @@ -2945,8 +2957,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest public void testUnionAllTablesWhenCastAndMappingIsRequired() { // Cannot plan this UNION ALL operation, because the column swap would require generating a subquery. - - msqCompatible(); assertQueryIsUnplannable( "SELECT\n" + "c, COUNT(*)\n" @@ -2961,6 +2971,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUnionAllSameTableTwice() { + notMsqCompatible(); testQuery( "SELECT\n" + "dim1, dim2, SUM(m1), COUNT(*)\n" @@ -3003,6 +3014,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUnionAllSameTableTwiceWithSameMapping() { + notMsqCompatible(); testQuery( "SELECT\n" + "dim1, dim2, SUM(m1), COUNT(*)\n" @@ -3046,8 +3058,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest public void testUnionAllSameTableTwiceWithDifferentMapping() { // Cannot plan this UNION ALL operation, because the column swap would require generating a subquery. - - msqCompatible(); assertQueryIsUnplannable( "SELECT\n" + "dim1, dim2, SUM(m1), COUNT(*)\n" @@ -3061,6 +3071,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUnionAllSameTableThreeTimes() { + notMsqCompatible(); testQuery( "SELECT\n" + "dim1, dim2, SUM(m1), COUNT(*)\n" @@ -3104,7 +3115,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUnionAllThreeTablesColumnCountMismatch1() { - msqCompatible(); try { testQuery( "SELECT\n" @@ -3129,7 +3139,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUnionAllThreeTablesColumnCountMismatch2() { - msqCompatible(); try { testQuery( "SELECT\n" @@ -3154,7 +3163,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUnionAllThreeTablesColumnCountMismatch3() { - msqCompatible(); try { testQuery( "SELECT\n" @@ -3179,6 +3187,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUnionAllSameTableThreeTimesWithSameMapping() { + notMsqCompatible(); testQuery( "SELECT\n" + "dim1, dim2, SUM(m1), COUNT(*)\n" @@ -3223,8 +3232,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest public void testPruneDeadAggregators() { // Test for ProjectAggregatePruneUnusedCallRule. - - msqCompatible(); testQuery( "SELECT\n" + " CASE 'foo'\n" @@ -3250,8 +3257,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest public void testPruneDeadAggregatorsThroughPostProjection() { // Test for ProjectAggregatePruneUnusedCallRule. - - msqCompatible(); testQuery( "SELECT\n" + " CASE 'foo'\n" @@ -3278,8 +3283,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest public void testPruneDeadAggregatorsThroughHaving() { // Test for ProjectAggregatePruneUnusedCallRule. - - msqCompatible(); testQuery( "SELECT\n" + " CASE 'foo'\n" @@ -3395,7 +3398,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testNullEmptyStringEquality() { - msqCompatible(); testQuery( "SELECT COUNT(*)\n" + "FROM druid.foo\n" @@ -3433,7 +3435,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testNullLongFilter() { - msqCompatible(); testQuery( "SELECT COUNT(*)\n" + "FROM druid.numfoo\n" @@ -3472,7 +3473,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testNullDoubleFilter() { - msqCompatible(); testQuery( "SELECT COUNT(*)\n" + "FROM druid.numfoo\n" @@ -3511,7 +3511,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testNullFloatFilter() { - msqCompatible(); testQuery( "SELECT COUNT(*)\n" + "FROM druid.numfoo\n" @@ -3550,8 +3549,10 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testNullDoubleTopN() { - // Disabled test in MSQ till https://github.com/apache/druid/issues/13951 is resolved - // msqCompatible(); + // Disabled test in MSQ with SQL-compatible mode till https://github.com/apache/druid/issues/13951 is resolved + if (NullHandling.sqlCompatible()) { + notMsqCompatible(); + } List expected; if (useDefault) { expected = ImmutableList.of( @@ -3592,8 +3593,10 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testNullFloatTopN() { - // Disabled test in MSQ till https://github.com/apache/druid/issues/13951 is resolved - // msqCompatible(); + // Disabled test in MSQ with SQL-compatible mode till https://github.com/apache/druid/issues/13951 is resolved + if (NullHandling.sqlCompatible()) { + notMsqCompatible(); + } List expected; if (useDefault) { expected = ImmutableList.of( @@ -3634,8 +3637,10 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testNullLongTopN() { - // Disabled test in MSQ till https://github.com/apache/druid/issues/13951 is resolved - // msqCompatible(); + // Disabled test in MSQ with SQL-compatible mode till https://github.com/apache/druid/issues/13951 is resolved + if (NullHandling.sqlCompatible()) { + notMsqCompatible(); + } List expected; if (useDefault) { expected = ImmutableList.of( @@ -3716,7 +3721,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testLongPredicateFilterNulls() { - msqCompatible(); testQuery( "SELECT COUNT(*)\n" + "FROM druid.numfoo\n" @@ -3738,7 +3742,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testDoublePredicateFilterNulls() { - msqCompatible(); testQuery( "SELECT COUNT(*)\n" + "FROM druid.numfoo\n" @@ -3760,7 +3763,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFloatPredicateFilterNulls() { - msqCompatible(); testQuery( "SELECT COUNT(*)\n" + "FROM druid.numfoo\n" @@ -3782,7 +3784,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testEmptyStringEquality() { - msqCompatible(); if (NullHandling.replaceWithDefault()) { testQuery( "SELECT COUNT(*)\n" @@ -3829,7 +3830,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testNullStringEquality() { - msqCompatible(); testQuery( "SELECT COUNT(*)\n" + "FROM druid.foo\n" @@ -3855,7 +3855,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCoalesceColumns() { - msqCompatible(); // Doesn't conform to the SQL standard, but it's how we do it. // This example is used in the sql.md doc. @@ -3900,7 +3899,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCoalesceColumnsFilter() { - msqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -3949,7 +3947,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCoalesceMoreColumns() { - msqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -4005,8 +4002,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest { // Doesn't conform to the SQL standard, but it's how we do it. // This example is used in the sql.md doc. - - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo WHERE dim2 IS NULL\n", ImmutableList.of( @@ -4028,7 +4023,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testSelfJoin() { - msqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -4079,6 +4073,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupingWithNullInFilter() { + notMsqCompatible(); testQuery( "SELECT COUNT(*) FROM foo WHERE dim1 IN (NULL)", ImmutableList.of( @@ -4111,7 +4106,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTwoExactCountDistincts() { - msqCompatible(); testQuery( PLANNER_CONFIG_NO_HLL, "SELECT COUNT(distinct dim1), COUNT(distinct dim2) FROM druid.foo", @@ -4188,6 +4182,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByNothingWithLiterallyFalseFilter() { + notMsqCompatible(); testQuery( "SELECT COUNT(*), MAX(cnt) FROM druid.foo WHERE 1 = 0", ImmutableList.of( @@ -4213,6 +4208,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByNothingWithImpossibleTimeFilter() { + notMsqCompatible(); // Regression test for https://github.com/apache/druid/issues/7671 testQuery( @@ -4237,7 +4233,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByWithImpossibleTimeFilter() { - msqCompatible(); // this gets optimized into 'false' testQuery( "SELECT dim1, COUNT(*) FROM druid.foo\n" @@ -4261,7 +4256,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByOneColumnWithLiterallyFalseFilter() { - msqCompatible(); testQuery( "SELECT COUNT(*), MAX(cnt) FROM druid.foo WHERE 1 = 0 GROUP BY dim1", ImmutableList.of( @@ -4285,6 +4279,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByWithFilterMatchingNothing() { + notMsqCompatible(); testQuery( "SELECT COUNT(*), MAX(cnt) FROM druid.foo WHERE dim1 = 'foobar'", ImmutableList.of( @@ -4309,7 +4304,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByWithGroupByEmpty() { - msqCompatible(); testQuery( "SELECT COUNT(*), SUM(cnt), MIN(cnt) FROM druid.foo GROUP BY ()", ImmutableList.of( @@ -4332,7 +4326,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByWithFilterMatchingNothingWithGroupByLiteral() { - msqCompatible(); testQuery( "SELECT COUNT(*), MAX(cnt) FROM druid.foo WHERE dim1 = 'foobar' GROUP BY 'dummy'", ImmutableList.of( @@ -4355,7 +4348,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountNonNullColumn() { - msqCompatible(); testQuery( "SELECT COUNT(cnt) FROM druid.foo", ImmutableList.of( @@ -4385,7 +4377,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountNullableColumn() { - msqCompatible(); testQuery( "SELECT COUNT(dim2) FROM druid.foo", ImmutableList.of( @@ -4415,7 +4406,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountNullableExpression() { - msqCompatible(); testQuery( "SELECT COUNT(CASE WHEN dim2 = 'abc' THEN 'yes' WHEN dim2 = 'def' THEN 'yes' END) FROM druid.foo", ImmutableList.of( @@ -4441,7 +4431,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStar() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo", ImmutableList.of( @@ -4462,7 +4451,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarOnCommonTableExpression() { - msqCompatible(); testQuery( "WITH beep (dim1_firstchar) AS (SELECT SUBSTRING(dim1, 1, 1) FROM foo WHERE dim2 = 'a')\n" + "SELECT COUNT(*) FROM beep WHERE dim1_firstchar <> 'z'", @@ -4488,7 +4476,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarOnView() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM view.aview WHERE dim1_firstchar <> 'z'", ImmutableList.of( @@ -4513,7 +4500,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testConfusedView() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM view.dview as druid WHERE druid.numfoo <> 'z'", ImmutableList.of( @@ -4538,6 +4524,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testViewAndJoin() { + notMsqCompatible(); cannotVectorize(); Map queryContext = withLeftDirectAccessEnabled(QUERY_CONTEXT_DEFAULT); testQuery( @@ -4589,7 +4576,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithLikeFilter() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo WHERE dim1 like 'a%' OR dim2 like '%xb%' escape 'x'", ImmutableList.of( @@ -4616,7 +4602,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithLongColumnFilters() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo WHERE cnt >= 3 OR cnt = 1", ImmutableList.of( @@ -4643,6 +4628,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithLongColumnFiltersOnFloatLiterals() { + notMsqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo WHERE cnt > 1.1 and cnt < 100000001.0", ImmutableList.of( @@ -4723,7 +4709,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithLongColumnFiltersOnTwoPoints() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo WHERE cnt = 1 OR cnt = 2", ImmutableList.of( @@ -4745,7 +4730,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilterOnStringAsNumber() { - msqCompatible(); testQuery( "SELECT distinct dim1 FROM druid.foo WHERE " + "dim1 = 10 OR " @@ -4784,7 +4768,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testSimpleLongAggregations() { - msqCompatible(); testQuery( "SELECT MIN(l1), MIN(cnt), MAX(l1) FROM druid.numfoo", ImmutableList.of( @@ -4809,7 +4792,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testSimpleDoubleAggregations() { - msqCompatible(); testQuery( "SELECT MIN(d1), MAX(d1) FROM druid.numfoo", ImmutableList.of( @@ -4833,7 +4815,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testSimpleFloatAggregations() { - msqCompatible(); testQuery( "SELECT MIN(m1), MAX(m1) FROM druid.numfoo", ImmutableList.of( @@ -5079,7 +5060,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilteredAggregations() { - msqCompatible(); testQuery( "SELECT " + "SUM(case dim1 when 'abc' then cnt end), " @@ -5178,7 +5158,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCaseFilteredAggregationWithGroupBy() { - msqCompatible(); testQuery( "SELECT\n" + " cnt,\n" @@ -5211,7 +5190,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilteredAggregationWithNotIn() { - msqCompatible(); testQuery( "SELECT\n" + "COUNT(*) filter(WHERE dim1 NOT IN ('1')),\n" @@ -5301,7 +5279,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testExpressionFilteringAndGrouping() { - msqCompatible(); testQuery( "SELECT\n" + " FLOOR(m1 / 2) * 2,\n" @@ -5348,7 +5325,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testExpressionFilteringAndGroupingUsingCastToLong() { - msqCompatible(); testQuery( "SELECT\n" + " CAST(m1 AS BIGINT) / 2 * 2,\n" @@ -5397,7 +5373,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testExpressionFilteringAndGroupingOnStringCastToNumber() { - msqCompatible(); testQuery( "SELECT\n" + " FLOOR(CAST(dim1 AS FLOAT) / 2) * 2,\n" @@ -5455,7 +5430,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testInFilter() { - msqCompatible(); testQuery( "SELECT dim1, COUNT(*) FROM druid.foo WHERE dim1 IN ('abc', 'def', 'ghi') GROUP BY dim1", ImmutableList.of( @@ -5516,8 +5490,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest public void testInFilterWith23Elements() { // Regression test for https://github.com/apache/druid/issues/4203. - - msqCompatible(); final List elements = new ArrayList<>(); elements.add("abc"); elements.add("def"); @@ -5555,7 +5527,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithDegenerateFilter() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo WHERE dim2 = 'a' and (dim1 > 'a' OR dim1 < 'b')", ImmutableList.of( @@ -5579,6 +5550,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithNotOfDegenerateFilter() { + notMsqCompatible(); // HashJoinSegmentStorageAdapter is not vectorizable cannotVectorize(); @@ -5607,6 +5579,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUnplannableQueries() { + notMsqCompatible(); // All of these queries are unplannable because they rely on features Druid doesn't support. // This test is here to confirm that we don't fall back to Calcite's interpreter or enumerable implementation. // It's also here so when we do support these features, we can have "real" tests for these queries. @@ -5635,7 +5608,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithBoundFilterSimplifyOnMetric() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo WHERE 2.5 < m1 AND m1 < 3.5", ImmutableList.of( @@ -5657,7 +5629,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithBoundFilterSimplifyOr() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo WHERE (dim1 >= 'a' and dim1 < 'b') OR dim1 = 'ab'", ImmutableList.of( @@ -5679,7 +5650,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUnplannableTwoExactCountDistincts() { - msqCompatible(); // Requires GROUPING SETS + GROUPING to be translated by AggregateExpandDistinctAggregatesRule. assertQueryIsUnplannable( @@ -5693,8 +5663,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest public void testUnplannableExactCountDistinctOnSketch() { // COUNT DISTINCT on a sketch cannot be exact. - - msqCompatible(); assertQueryIsUnplannable( PLANNER_CONFIG_NO_HLL, "SELECT COUNT(distinct unique_dim1) FROM druid.foo", @@ -5710,6 +5678,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testArrayAggQueryOnComplexDatatypes() { + notMsqCompatible(); cannotVectorize(); testQuery( "SELECT ARRAY_AGG(unique_dim1) FROM druid.foo", @@ -5748,7 +5717,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testStringAggQueryOnComplexDatatypes() { - msqCompatible(); try { testQuery("SELECT STRING_AGG(unique_dim1, ',') FROM druid.foo", ImmutableList.of(), ImmutableList.of()); Assert.fail("query execution should fail"); @@ -5765,7 +5733,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithBoundFilterSimplifyAnd() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo WHERE (dim1 >= 'a' and dim1 < 'b') and dim1 = 'abc'", ImmutableList.of( @@ -5787,7 +5754,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithFilterOnCastedString() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo WHERE CAST(dim1 AS bigint) = 2", ImmutableList.of( @@ -5809,7 +5775,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithTimeFilter() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo " + "WHERE __time >= TIMESTAMP '2000-01-01 00:00:00' AND __time < TIMESTAMP '2001-01-01 00:00:00'", @@ -5831,7 +5796,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithTimeInIntervalFilter() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo " + "WHERE TIME_IN_INTERVAL(__time, '2000-01-01/P1Y') " @@ -5854,7 +5818,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithTimeInIntervalFilterLosAngeles() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo " + "WHERE TIME_IN_INTERVAL(__time, '2000-01-01/P1Y')", @@ -5877,6 +5840,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithTimeInIntervalFilterInvalidInterval() { + notMsqCompatible(); testQueryThrows( "SELECT COUNT(*) FROM druid.foo " + "WHERE TIME_IN_INTERVAL(__time, '2000-01-01/X')", @@ -5893,6 +5857,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithTimeInIntervalFilterNonLiteral() { + notMsqCompatible(); testQueryThrows( "SELECT COUNT(*) FROM druid.foo " + "WHERE TIME_IN_INTERVAL(__time, dim1)", @@ -5909,7 +5874,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithBetweenTimeFilterUsingMilliseconds() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo " + "WHERE __time BETWEEN TIMESTAMP '2000-01-01 00:00:00' AND TIMESTAMP '2000-12-31 23:59:59.999'", @@ -5931,7 +5895,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithBetweenTimeFilterUsingMillisecondsInStringLiterals() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo " + "WHERE __time BETWEEN '2000-01-01 00:00:00' AND '2000-12-31 23:59:59.999'", @@ -5953,7 +5916,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testRemoveUselessCaseWhen() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo\n" + "WHERE\n" @@ -5982,7 +5944,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithTimeMillisecondFilters() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo\n" + "WHERE __time = TIMESTAMP '2000-01-01 00:00:00.111'\n" @@ -6011,8 +5972,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest public void testCountStarWithTimeFilterUsingStringLiterals() { // Strings are implicitly cast to timestamps. Test a few different forms. - - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo\n" + "WHERE __time >= '2000-01-01 00:00:00' AND __time < '2001-01-01T00:00:00'\n" @@ -6042,7 +6001,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithTimeFilterUsingStringLiteralsInvalid_isUnplannable() { - msqCompatible(); // Strings are implicitly cast to timestamps. Test an invalid string. // This error message isn't ideal but it is at least better than silently ignoring the problem. assertQueryIsUnplannable( @@ -6055,7 +6013,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithSinglePointInTime() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo WHERE __time = TIMESTAMP '2000-01-01 00:00:00'", ImmutableList.of( @@ -6076,7 +6033,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithTwoPointsInTime() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo WHERE " + "__time = TIMESTAMP '2000-01-01 00:00:00' OR __time = TIMESTAMP '2000-01-01 00:00:00' + INTERVAL '1' DAY", @@ -6103,7 +6059,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithComplexDisjointTimeFilter() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo " + "WHERE dim2 = 'a' and (" @@ -6144,7 +6099,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithNotOfComplexDisjointTimeFilter() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo " + "WHERE not (dim2 = 'a' and (" @@ -6186,7 +6140,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithNotTimeFilter() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo " + "WHERE dim1 <> 'xxx' and not (" @@ -6217,7 +6170,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithTimeAndDimFilter() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo " + "WHERE dim2 <> 'a' " @@ -6241,7 +6193,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithTimeOrDimFilter() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo " + "WHERE dim2 <> 'a' " @@ -6278,7 +6229,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithTimeFilterOnLongColumnUsingExtractEpoch() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo WHERE " + "cnt >= EXTRACT(EPOCH FROM TIMESTAMP '1970-01-01 00:00:00') * 1000 " @@ -6312,7 +6262,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithTimeFilterOnLongColumnUsingExtractEpochFromDate() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo WHERE " + "cnt >= EXTRACT(EPOCH FROM DATE '1970-01-01') * 1000 " @@ -6346,7 +6295,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountStarWithTimeFilterOnLongColumnUsingTimestampToMillis() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo WHERE " + "cnt >= TIMESTAMP_TO_MILLIS(TIMESTAMP '1970-01-01 00:00:00') " @@ -6380,7 +6328,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testSumOfString() { - msqCompatible(); testQuery( "SELECT SUM(CAST(dim1 AS INTEGER)) FROM druid.foo", ImmutableList.of( @@ -6411,7 +6358,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testSumOfExtractionFn() { - msqCompatible(); // Cannot vectorize due to expressions in aggregators. cannotVectorize(); @@ -6445,7 +6391,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeseriesWithTimeFilterOnLongColumnUsingMillisToTimestamp() { - msqCompatible(); testQuery( "SELECT\n" + " FLOOR(MILLIS_TO_TIMESTAMP(cnt) TO YEAR),\n" @@ -6490,6 +6435,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountDistinct() { + notMsqCompatible(); testQuery( "SELECT SUM(cnt), COUNT(distinct dim2), COUNT(distinct unique_dim1) FROM druid.foo", ImmutableList.of( @@ -6522,7 +6468,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountDistinctOfCaseWhen() { - msqCompatible(); testQuery( "SELECT\n" + "COUNT(DISTINCT CASE WHEN m1 >= 4 THEN m1 END),\n" @@ -6574,7 +6519,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testExactCountDistinct() { - msqCompatible(); // When HLL is disabled, do exact count distinct through a nested query. testQuery( @@ -6614,10 +6558,12 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testApproxCountDistinctWhenHllDisabled() { - // When HLL is disabled, APPROX_COUNT_DISTINCT is still approximate. + if (NullHandling.sqlCompatible()) { + // Empty string isn't counted properly; see https://github.com/apache/druid/issues/13950 + notMsqCompatible(); + } - // Disabled test in MSQ till https://github.com/apache/druid/issues/13950 is resolved - // msqCompatible(); + // When HLL is disabled, APPROX_COUNT_DISTINCT is still approximate. testQuery( PLANNER_CONFIG_NO_HLL, "SELECT APPROX_COUNT_DISTINCT(dim2) FROM druid.foo", @@ -6650,8 +6596,11 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testApproxCountDistinctBuiltin() { - // Disabled test in MSQ till https://github.com/apache/druid/issues/13950 is resolved - // msqCompatible(); + if (NullHandling.sqlCompatible()) { + // Empty string isn't counted properly; see https://github.com/apache/druid/issues/13950 + notMsqCompatible(); + } + testQuery( "SELECT APPROX_COUNT_DISTINCT_BUILTIN(dim2) FROM druid.foo", ImmutableList.of( @@ -6736,6 +6685,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testMultipleExactCountDistinctWithGroupingAndOtherAggregators() { + notMsqCompatible(); requireMergeBuffers(4); testQuery( PLANNER_CONFIG_NO_HLL.withOverrides( @@ -6812,6 +6762,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testApproxCountDistinct() { + notMsqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -6892,7 +6843,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testApproxCountDistinctOnVectorizableSingleStringExpression() { - msqCompatible(); testQuery( "SELECT APPROX_COUNT_DISTINCT(dim1 || 'hello') FROM druid.foo", ImmutableList.of( @@ -6924,7 +6874,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testNestedGroupBy() { - msqCompatible(); testQuery( "SELECT\n" + " FLOOR(__time to hour) AS __time,\n" @@ -6997,7 +6946,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testDoubleNestedGroupBy() { - msqCompatible(); requireMergeBuffers(3); testQuery( "SELECT SUM(cnt), COUNT(*) FROM (\n" @@ -7055,8 +7003,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest { // This test fails when AggregateMergeRule is added to Rules.ABSTRACT_RELATIONAL_RULES. So, we don't add that // rule for now. Possible bug in the rule. - - msqCompatible(); testQuery( "SELECT MAX(cnt) FROM (\n" + " SELECT dim2, MAX(t1.cnt) cnt FROM (\n" @@ -7106,7 +7052,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testExactCountDistinctUsingSubquery() { - msqCompatible(); testQuery( "SELECT\n" + " SUM(cnt),\n" @@ -7148,6 +7093,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testExactCountDistinctUsingSubqueryOnUnionAllTables() { + notMsqCompatible(); testQuery( "SELECT\n" + " SUM(cnt),\n" @@ -7200,7 +7146,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUseTimeFloorInsteadOfGranularityOnJoinResult() { - msqCompatible(); cannotVectorize(); testQuery( @@ -7427,7 +7372,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testExactCountDistinctOfSemiJoinResult() { - msqCompatible(); // Cannot vectorize due to extraction dimension spec. cannotVectorize(); @@ -7497,6 +7441,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testMaxSubqueryRows() { + notMsqCompatible(); expectedException.expect(ResourceLimitExceededException.class); expectedException.expectMessage("Subquery generated results beyond maximum[2]"); @@ -7517,7 +7462,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testZeroMaxNumericInFilter() { - msqCompatible(); expectedException.expect(UOE.class); expectedException.expectMessage("[maxNumericInFilters] must be greater than 0"); @@ -7538,7 +7482,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testHighestMaxNumericInFilter() { - msqCompatible(); expectedException.expect(UOE.class); expectedException.expectMessage("Expected parameter[maxNumericInFilters] cannot exceed system set value of [100]"); @@ -7559,6 +7502,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testQueryWithMoreThanMaxNumericInFilter() { + notMsqCompatible(); expectedException.expect(UOE.class); expectedException.expectMessage("The number of values in the IN clause for [dim6] in query exceeds configured maxNumericFilter limit of [2] for INs. Cast [3] values of IN clause to String"); @@ -7579,7 +7523,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testExactCountDistinctUsingSubqueryWithWherePushDown() { - msqCompatible(); testQuery( "SELECT\n" + " SUM(cnt),\n" @@ -7662,7 +7605,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testExactCountDistinctUsingSubqueryWithWhereToOuterFilter() { - msqCompatible(); // Cannot vectorize topN operator. cannotVectorize(); @@ -7760,7 +7702,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testHistogramUsingSubquery() { - msqCompatible(); testQuery( "SELECT\n" + " CAST(thecnt AS VARCHAR),\n" @@ -7806,7 +7747,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testHistogramUsingSubqueryWithSort() { - msqCompatible(); testQuery( "SELECT\n" + " CAST(thecnt AS VARCHAR),\n" @@ -7861,6 +7801,11 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountDistinctArithmetic() { + if (NullHandling.sqlCompatible()) { + // Empty string isn't counted properly; see https://github.com/apache/druid/issues/13950 + notMsqCompatible(); + } + testQuery( "SELECT\n" + " SUM(cnt),\n" @@ -7905,7 +7850,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCountDistinctOfSubstring() { - msqCompatible(); // Cannot vectorize due to extraction dimension spec. cannotVectorize(); @@ -7947,8 +7891,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest public void testCountDistinctOfTrim() { // Test a couple different syntax variants of TRIM. - - msqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -8068,7 +8010,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testRegexpExtractFilterViaNotNullCheck() { - msqCompatible(); // Cannot vectorize due to extractionFn in dimension spec. cannotVectorize(); @@ -8103,7 +8044,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testRegexpLikeFilter() { - msqCompatible(); testQuery( "SELECT COUNT(*)\n" + "FROM foo\n" @@ -8176,10 +8116,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByLimitPushDownWithHavingOnLong() { - if (NullHandling.sqlCompatible()) { - msqCompatible(); - } - testQuery( "SELECT dim1, dim2, SUM(cnt) AS thecnt " + "FROM druid.foo " @@ -8234,7 +8170,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByLimitPushdownExtraction() { - msqCompatible(); cannotVectorize(); testQuery( @@ -8276,7 +8211,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilterOnTimeFloor() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo\n" + "WHERE\n" @@ -8300,7 +8234,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupAndFilterOnTimeFloorWithTimeZone() { - msqCompatible(); testQuery( "SELECT TIME_FLOOR(__time, 'P1M', NULL, 'America/Los_Angeles'), COUNT(*)\n" + "FROM druid.foo\n" @@ -8334,7 +8267,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilterOnCurrentTimestampWithIntervalArithmetic() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo\n" + "WHERE\n" @@ -8358,7 +8290,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilterOnCurrentTimestampLosAngeles() { - msqCompatible(); testQuery( PLANNER_CONFIG_DEFAULT, QUERY_CONTEXT_LOS_ANGELES, @@ -8383,7 +8314,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilterOnCurrentTimestampOnView() { - msqCompatible(); testQuery( "SELECT * FROM view.bview", ImmutableList.of( @@ -8406,8 +8336,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest { // Tests that query context still applies to view SQL; note the result is different from // "testFilterOnCurrentTimestampOnView" above. - - msqCompatible(); testQuery( PLANNER_CONFIG_DEFAULT, QUERY_CONTEXT_LOS_ANGELES, @@ -8431,7 +8359,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilterOnNotTimeFloor() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo\n" + "WHERE\n" @@ -8457,7 +8384,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilterOnTimeFloorComparison() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo\n" + "WHERE\n" @@ -8480,7 +8406,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilterOnTimeFloorComparisonMisaligned() { - msqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo\n" + "WHERE\n" @@ -8503,7 +8428,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilterOnTimeExtract() { - msqCompatible(); // Cannot vectorize due to expression filter. cannotVectorize(); @@ -8539,7 +8463,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilterOnTimeExtractWithMultipleDays() { - msqCompatible(); // Cannot vectorize due to expression filters. cannotVectorize(); @@ -8583,6 +8506,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilterOnTimeExtractWithVariousTimeUnits() { + notMsqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -8645,6 +8569,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilterOnTimeFloorMisaligned() { + notMsqCompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo " + "WHERE floor(__time TO month) = TIMESTAMP '2000-01-01 00:00:01'", @@ -8664,7 +8589,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByFloor() { - msqCompatible(); testQuery( "SELECT floor(CAST(dim1 AS float)), COUNT(*) FROM druid.foo GROUP BY floor(CAST(dim1 AS float))", ImmutableList.of( @@ -8692,6 +8616,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testQueryWithSelectProjectAndIdentityProjectDoesNotRename() { + notMsqCompatible(); cannotVectorize(); requireMergeBuffers(3); testQuery( @@ -8791,7 +8716,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByFloorWithOrderBy() { - msqCompatible(); testQuery( "SELECT floor(CAST(dim1 AS float)) AS fl, COUNT(*) FROM druid.foo GROUP BY floor(CAST(dim1 AS float)) ORDER BY fl DESC", ImmutableList.of( @@ -9004,8 +8928,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest { // Cannot vectorize due to extraction dimension spec. cannotVectorize(); - msqCompatible(); - final RegisteredLookupExtractionFn extractionFn = new RegisteredLookupExtractionFn( null, "lookyloo", @@ -9043,6 +8965,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByExpressionFromLookup() { + notMsqCompatible(); // Cannot vectorize direct queries on lookup tables. cannotVectorize(); @@ -9076,7 +8999,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeseries() { - msqCompatible(); testQuery( "SELECT SUM(cnt), gran FROM (\n" + " SELECT floor(__time TO month) AS gran,\n" @@ -9103,7 +9025,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilteredTimeAggregators() { - msqCompatible(); testQuery( "SELECT\n" + " SUM(cnt) FILTER(WHERE __time >= TIMESTAMP '2000-01-01 00:00:00'\n" @@ -9236,7 +9157,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeseriesUsingTimeFloor() { - msqCompatible(); testQuery( "SELECT SUM(cnt), gran FROM (\n" + " SELECT TIME_FLOOR(__time, 'P1M') AS gran,\n" @@ -9263,7 +9183,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeseriesUsingTimeFloorWithTimeShift() { - msqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -9304,7 +9223,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeseriesUsingTimeFloorWithTimestampAdd() { - msqCompatible(); testQuery( "SELECT SUM(cnt), gran FROM (\n" + " SELECT TIME_FLOOR(TIMESTAMPADD(DAY, -1, __time), 'P1M') AS gran,\n" @@ -9342,7 +9260,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeseriesUsingTimeFloorWithOrigin() { - msqCompatible(); testQuery( "SELECT SUM(cnt), gran FROM (\n" + " SELECT TIME_FLOOR(__time, 'P1M', TIMESTAMP '1970-01-01 01:02:03') AS gran,\n" @@ -9377,7 +9294,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeseriesLosAngelesUsingTimeFloorConnectionUtc() { - msqCompatible(); testQuery( "SELECT SUM(cnt), gran FROM (\n" + " SELECT TIME_FLOOR(__time, 'P1M', CAST(NULL AS TIMESTAMP), 'America/Los_Angeles') AS gran,\n" @@ -9437,6 +9353,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeseriesDontSkipEmptyBuckets() { + notMsqCompatible(); // Tests that query context parameters are passed through to the underlying query engine. Long defaultVal = NullHandling.replaceWithDefault() ? 0L : null; testQuery( @@ -9548,7 +9465,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest { // Cannot vectorize due to descending order. cannotVectorize(); - msqCompatible(); testQuery( "SELECT gran, SUM(cnt) FROM (\n" @@ -9577,6 +9493,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeseriesEmptyResultsAggregatorDefaultValues() { + notMsqCompatible(); // timeseries with all granularity have a single group, so should return default results for given aggregators testQuery( "SELECT\n" @@ -9682,6 +9599,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeseriesEmptyResultsAggregatorDefaultValuesNonVectorized() { + notMsqCompatible(); cannotVectorize(); // timeseries with all granularity have a single group, so should return default results for given aggregators testQuery( @@ -9968,6 +9886,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByAggregatorDefaultValuesNonVectorized() { + notMsqCompatible(); cannotVectorize(); testQuery( "SELECT\n" @@ -10132,7 +10051,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByExtractYear() { - msqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -10171,7 +10089,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByFormatYearAndMonth() { - msqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -10210,7 +10127,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByExtractFloorTime() { - msqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -10248,7 +10164,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest { // Cannot vectorize due to virtual columns. cannotVectorize(); - msqCompatible(); testQuery( PLANNER_CONFIG_DEFAULT, @@ -10286,7 +10201,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeseriesWithLimitNoTopN() { - msqCompatible(); testQuery( PLANNER_CONFIG_NO_TOPN, "SELECT gran, SUM(cnt)\n" @@ -10317,7 +10231,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeseriesWithLimit() { - msqCompatible(); testQuery( "SELECT gran, SUM(cnt)\n" + "FROM (\n" @@ -10346,7 +10259,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest public void testTimeseriesWithLimitAndOffset() { // Timeseries cannot handle offsets, so the query morphs into a groupBy. - msqCompatible(); testQuery( "SELECT gran, SUM(cnt)\n" + "FROM (\n" @@ -10383,7 +10295,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeseriesWithOrderByAndLimit() { - msqCompatible(); testQuery( "SELECT gran, SUM(cnt)\n" + "FROM (\n" @@ -10564,6 +10475,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupingSets() { + notMsqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -10628,6 +10540,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupingAggregatorDifferentOrder() { + notMsqCompatible(); requireMergeBuffers(3); testQuery( @@ -10691,6 +10604,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupingAggregatorWithPostAggregator() { + notMsqCompatible(); List resultList; if (NullHandling.sqlCompatible()) { resultList = ImmutableList.of( @@ -10749,6 +10663,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupingSetsWithNumericDimension() { + notMsqCompatible(); testQuery( "SELECT cnt, COUNT(*)\n" + "FROM foo\n" @@ -10779,6 +10694,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByRollup() { + notMsqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -10837,6 +10753,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByRollupDifferentOrder() { + notMsqCompatible(); // Like "testGroupByRollup", but the ROLLUP exprs are in the reverse order. testQuery( "SELECT dim2, gran, SUM(cnt)\n" @@ -10892,6 +10809,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupByCube() { + notMsqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -10953,6 +10871,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupingSetsWithDummyDimension() { + notMsqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -11014,6 +10933,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupingSetsNoSuperset() { + notMsqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -11070,6 +10990,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupingSetsWithOrderByDimension() { + notMsqCompatible(); testQuery( "SELECT dim2, gran, SUM(cnt)\n" + "FROM (SELECT FLOOR(__time TO MONTH) AS gran, COALESCE(dim2, '') dim2, cnt FROM druid.foo) AS x\n" @@ -11140,6 +11061,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupingSetsWithOrderByAggregator() { + notMsqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -11208,6 +11130,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupingSetsWithOrderByAggregatorWithLimit() { + notMsqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -11273,7 +11196,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest public void testTimeExtractWithTooFewArguments() { // Regression test for https://github.com/apache/druid/pull/7710. - msqCompatible(); try { testQuery("SELECT TIME_EXTRACT(__time) FROM druid.foo", ImmutableList.of(), ImmutableList.of()); Assert.fail("query execution should fail"); @@ -11290,7 +11212,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUsingSubqueryAsFilterOnTwoColumns() { - msqCompatible(); testQuery( "SELECT __time, cnt, dim1, dim2 FROM druid.foo " + " WHERE (dim1, dim2) IN (" @@ -11402,7 +11323,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUsingSubqueryWithLimit() { - msqCompatible(); // Cannot vectorize scan query. cannotVectorize(); @@ -11434,7 +11354,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUsingSubqueryWithoutLimit() { - msqCompatible(); testQuery( "SELECT COUNT(*) AS cnt FROM ( SELECT * FROM druid.foo ) tmpA", ImmutableList.of( @@ -11455,7 +11374,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testUnicodeFilterAndGroupBy() { - msqCompatible(); testQuery( "SELECT\n" + " dim1,\n" @@ -11493,7 +11411,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testOrderByAlongWithAliasOrderByTimeGroupByMulti() { - msqCompatible(); testQuery( "select __time as bug, dim2 from druid.foo group by 1, 2 order by 1 limit 1", ImmutableList.of( @@ -11527,7 +11444,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testOrderByAlongWithAliasOrderByTimeGroupByOneCol() { - msqCompatible(); testQuery( "select __time as bug from druid.foo group by 1 order by 1 limit 1", ImmutableList.of( @@ -11621,6 +11537,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest + " org.apache.calcite.sql.validate.SqlValidatorException: Column 'dim1' is ambiguous") public void testProjectAfterSort3() { + notMsqCompatible(); testQuery( "select dim1 from (select dim1, dim1, count(*) cnt from druid.foo group by dim1, dim1 order by cnt)", ImmutableList.of( @@ -11659,6 +11576,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testProjectAfterSort3WithoutAmbiguity() { + notMsqCompatible(); // This query is equivalent to the one in testProjectAfterSort3 but renames the second grouping column // to avoid the ambiguous name exception. The inner sort is also optimized out in Calcite 1.21. testQuery( @@ -11690,7 +11608,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testSortProjectAfterNestedGroupBy() { - msqCompatible(); testQuery( "SELECT " + " cnt " @@ -11762,7 +11679,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testPostAggWithTimeseries() { - msqCompatible(); // Cannot vectorize due to descending order. cannotVectorize(); @@ -11806,7 +11722,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testPostAggWithTopN() { - msqCompatible(); testQuery( "SELECT " + " AVG(m2), " @@ -11869,7 +11784,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testConcat() { - msqCompatible(); testQuery( "SELECT CONCAT(dim1, '-', dim1, '_', dim1) as dimX FROM foo", ImmutableList.of( @@ -11926,7 +11840,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testConcatGroup() { - msqCompatible(); testQuery( "SELECT CONCAT(dim1, '-', dim1, '_', dim1) as dimX FROM foo GROUP BY 1", ImmutableList.of( @@ -12048,7 +11961,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testRequireTimeConditionPositive() { - msqCompatible(); // simple timeseries testQuery( PLANNER_CONFIG_REQUIRE_TIME_CONDITION, @@ -12214,6 +12126,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testRequireTimeConditionSimpleQueryNegative() { + notMsqCompatible(); expectedException.expect(CannotBuildQueryException.class); expectedException.expectMessage("__time column"); @@ -12234,6 +12147,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testRequireTimeConditionSubQueryNegative() { + notMsqCompatible(); expectedException.expect(CannotBuildQueryException.class); expectedException.expectMessage("__time column"); @@ -12252,6 +12166,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testRequireTimeConditionSemiJoinNegative() { + notMsqCompatible(); expectedException.expect(CannotBuildQueryException.class); expectedException.expectMessage("__time column"); @@ -12271,7 +12186,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilterFloatDimension() { - msqCompatible(); testQuery( "SELECT dim1 FROM numfoo WHERE f1 = 0.1 LIMIT 1", ImmutableList.of( @@ -12294,7 +12208,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testFilterDoubleDimension() { - msqCompatible(); testQuery( "SELECT dim1 FROM numfoo WHERE d1 = 1.7 LIMIT 1", ImmutableList.of( @@ -12339,6 +12252,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTrigonometricFunction() { + notMsqCompatible(); testQuery( PLANNER_CONFIG_DEFAULT, QUERY_CONTEXT_DEFAULT, @@ -12388,7 +12302,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testRadiansAndDegrees() { - msqCompatible(); testQuery( "SELECT RADIANS(m1 * 15)/DEGREES(m2) FROM numfoo WHERE dim1 = '1'", ImmutableList.of( @@ -12465,7 +12378,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimestampCeil() { - msqCompatible(); testQuery( "SELECT CEIL(TIMESTAMP '2000-01-01 00:00:00' TO DAY), \n" + "CEIL(TIMESTAMP '2000-01-01 01:00:00' TO DAY) \n" @@ -12504,8 +12416,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testNvlColumns() { - msqCompatible(); - testQuery( "SELECT NVL(dim2, dim1), COUNT(*) FROM druid.foo GROUP BY NVL(dim2, dim1)\n", ImmutableList.of( @@ -12641,6 +12551,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testQueryContextOuterLimit() { + notMsqCompatible(); Map outerLimitContext = new HashMap<>(QUERY_CONTEXT_DEFAULT); outerLimitContext.put(PlannerContext.CTX_SQL_OUTER_LIMIT, 4); @@ -12763,7 +12674,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testValidationErrorNullLiteralIllegal() { - msqCompatible(); expectedException.expectMessage("Illegal use of 'NULL'"); testQuery( @@ -12776,7 +12686,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testValidationErrorNonLiteralIllegal() { - msqCompatible(); expectedException.expectMessage("Argument to function 'REGEXP_LIKE' must be a literal"); testQuery( @@ -12789,7 +12698,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testValidationErrorWrongTypeLiteral() { - msqCompatible(); expectedException.expectMessage("Cannot apply 'REGEXP_LIKE' to arguments"); testQuery( @@ -12802,7 +12710,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeStampAddZeroDayPeriod() { - msqCompatible(); testQuery( "SELECT TIMESTAMPADD(DAY, 0, \"__time\") FROM druid.foo", @@ -12828,7 +12735,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeStampAddZeroMonthPeriod() { - msqCompatible(); testQuery( "SELECT TIMESTAMPADD(MONTH, 0, \"__time\") FROM druid.foo", @@ -12858,7 +12764,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeStampAddZeroYearPeriod() { - msqCompatible(); skipVectorize(); testQuery( @@ -12894,7 +12799,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeStampAddConversion() { - msqCompatible(); final PeriodGranularity periodGranularity = new PeriodGranularity(new Period("P1M"), null, null); // @@ -12958,6 +12862,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupingSetsWithLimit() { + notMsqCompatible(); testQuery( "SELECT dim2, gran, SUM(cnt)\n" + "FROM (SELECT FLOOR(__time TO MONTH) AS gran, COALESCE(dim2, '') dim2, cnt FROM druid.foo) AS x\n" @@ -13022,6 +12927,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGroupingSetsWithLimitOrderByGran() { + notMsqCompatible(); // Cannot vectorize due to virtual columns. cannotVectorize(); @@ -13133,8 +13039,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testRoundFunc() { - - msqCompatible(); testQuery( "SELECT f1, round(f1) FROM druid.numfoo", ImmutableList.of( @@ -13246,6 +13150,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testEmptyGroupWithOffsetDoesntInfiniteLoop() { + notMsqCompatible(); testQuery( "SELECT r0.c, r1.c\n" + "FROM (\n" @@ -13307,7 +13212,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testJoinWithTimeDimension() { - msqCompatible(); testQuery( PLANNER_CONFIG_DEFAULT, QUERY_CONTEXT_DEFAULT, @@ -13344,7 +13248,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testExpressionCounts() { - msqCompatible(); cannotVectorize(); testQuery( "SELECT\n" @@ -13394,7 +13297,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testBitwiseAggregatorsTimeseries() { - msqCompatible(); cannotVectorize(); testQuery( "SELECT\n" @@ -13676,7 +13578,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testStringAggMultiValue() { - msqCompatible(); cannotVectorize(); testQuery( "SELECT STRING_AGG(dim3, ','), STRING_AGG(DISTINCT dim3, ',') FROM foo", @@ -13741,7 +13642,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testStringAggNumeric() { - msqCompatible(); cannotVectorize(); testQuery( "SELECT STRING_AGG(l1, ','), STRING_AGG(DISTINCT l1, ','), STRING_AGG(d1, ','), STRING_AGG(DISTINCT d1, ','), STRING_AGG(f1, ','), STRING_AGG(DISTINCT f1, ',') FROM numfoo", @@ -13896,7 +13796,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testStringAggExpression() { - msqCompatible(); cannotVectorize(); testQuery( "SELECT STRING_AGG(DISTINCT CONCAT(dim1, dim2), ','), STRING_AGG(DISTINCT CONCAT(dim1, dim2), CONCAT('|', '|')) FROM foo", @@ -13964,7 +13863,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test(expected = RelOptPlanner.CannotPlanException.class) public void testStringAggExpressionNonConstantSeparator() { - msqCompatible(); testQuery( "SELECT STRING_AGG(DISTINCT CONCAT(dim1, dim2), CONCAT('|', dim1)) FROM foo", ImmutableList.of(), @@ -13975,7 +13873,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testStringAggMaxBytes() { - msqCompatible(); cannotVectorize(); testQuery( "SELECT STRING_AGG(l1, ',', 128), STRING_AGG(DISTINCT l1, ',', 128) FROM numfoo", @@ -14044,7 +13941,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testHumanReadableFormatFunction() { - msqCompatible(); // For the row where dim1 = '1', m1 = 4.0 and l1 is null testQuery( "SELECT m1, " @@ -14113,7 +14009,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testHumanReadableFormatFunctionExceptionWithWrongNumberType() { - msqCompatible(); this.expectedException.expect(SqlPlanningException.class); this.expectedException.expectMessage("Supported form(s): HUMAN_READABLE_BINARY_BYTE_FORMAT(Number, [Precision])"); testQuery( @@ -14126,7 +14021,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testHumanReadableFormatFunctionWithWrongPrecisionType() { - msqCompatible(); this.expectedException.expect(SqlPlanningException.class); this.expectedException.expectMessage("Supported form(s): HUMAN_READABLE_BINARY_BYTE_FORMAT(Number, [Precision])"); testQuery( @@ -14139,7 +14033,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testHumanReadableFormatFunctionWithInvalidNumberOfArguments() { - msqCompatible(); this.expectedException.expect(SqlPlanningException.class); /* @@ -14162,7 +14055,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testCommonVirtualExpressionWithDifferentValueType() { - msqCompatible(); testQuery( "select\n" + " dim1,\n" @@ -14203,7 +14095,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testReturnEmptyRowWhenGroupByIsConvertedToTimeseriesWithSingleConstantDimension() { - msqCompatible(); skipVectorize(); testQuery( "SELECT 'A' from foo WHERE m1 = 50 AND dim1 = 'wat' GROUP BY 'foobar'", @@ -14256,7 +14147,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testReturnEmptyRowWhenGroupByIsConvertedToTimeseriesWithMultipleConstantDimensions() { - msqCompatible(); skipVectorize(); testQuery( "SELECT 'A', dim1 from foo WHERE m1 = 50 AND dim1 = 'wat' GROUP BY dim1", @@ -14310,7 +14200,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testPlanWithInFilterLessThanInSubQueryThreshold() { - msqCompatible(); String query = "SELECT l1 FROM numfoo WHERE l1 IN (4842, 4844, 4845, 14905, 4853, 29064)"; testQuery( @@ -14412,6 +14301,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testGreatestFunctionForStringWithIsNull() { + notMsqCompatible(); cannotVectorize(); String query = "SELECT l1, LATEST(GREATEST(dim1, dim2)) IS NULL FROM druid.numfoo GROUP BY l1"; @@ -14501,7 +14391,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testTimeseriesQueryWithEmptyInlineDatasourceAndGranularity() { - msqCompatible(); // the SQL query contains an always FALSE filter ('bar' = 'baz'), which optimizes the query to also remove time // filter. the converted query hence contains ETERNITY interval but still a MONTH granularity due to the grouping. // Such a query should plan into a GroupBy query with a timestamp_floor function, instead of a timeseries @@ -14585,6 +14474,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest @Test public void testComplexDecodeAgg() { + notMsqCompatible(); cannotVectorize(); testQuery( "SELECT APPROX_COUNT_DISTINCT_BUILTIN(COMPLEX_DECODE_BASE64('hyperUnique',PARSE_JSON(TO_JSON_STRING(unique_dim1)))) from druid.foo", diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/run/NativeQueryMakerTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/run/SqlResultsTest.java similarity index 95% rename from sql/src/test/java/org/apache/druid/sql/calcite/run/NativeQueryMakerTest.java rename to sql/src/test/java/org/apache/druid/sql/calcite/run/SqlResultsTest.java index e44c6868ec2..d19e218075a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/run/NativeQueryMakerTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/run/SqlResultsTest.java @@ -27,7 +27,7 @@ import org.junit.Test; import java.util.Arrays; import java.util.List; -public class NativeQueryMakerTest +public class SqlResultsTest { @Test @@ -105,12 +105,12 @@ public class NativeQueryMakerTest @Test public void testMustCoerce() { - Assert.assertNull(NativeQueryMaker.maybeCoerceArrayToList("hello", true)); + Assert.assertNull(SqlResults.maybeCoerceArrayToList("hello", true)); } private static void assertCoerced(Object expected, Object toCoerce, boolean mustCoerce) { - Object coerced = NativeQueryMaker.maybeCoerceArrayToList(toCoerce, mustCoerce); + Object coerced = SqlResults.maybeCoerceArrayToList(toCoerce, mustCoerce); Assert.assertEquals(expected, coerced); } }