From c98c66558f7d5ad48edde681ba5192130268b24a Mon Sep 17 00:00:00 2001 From: Abhishek Radhakrishnan Date: Mon, 17 Apr 2023 08:30:25 -0700 Subject: [PATCH] Include statement attributes in `EXPLAIN PLAN` output (#14074) This commit adds attributes that contain metadata information about the query in the EXPLAIN PLAN output. The attributes currently contain two items: - `statementTyp`: SELECT, INSERT or REPLACE - `targetDataSource`: provides the target datasource name for DML statements It is added to both the legacy and native query plan outputs. --- docs/querying/sql-translation.md | 220 ++++++++++-------- docs/querying/sql.md | 4 +- .../sql/calcite/planner/DruidPlanner.java | 1 + .../calcite/planner/ExplainAttributes.java | 73 ++++++ .../sql/calcite/planner/IngestHandler.java | 18 ++ .../sql/calcite/planner/PlannerContext.java | 16 ++ .../sql/calcite/planner/QueryHandler.java | 26 ++- .../calcite/planner/SqlStatementHandler.java | 1 + .../sql/avatica/DruidAvaticaHandlerTest.java | 4 +- .../sql/calcite/BaseCalciteQueryTest.java | 3 + .../sql/calcite/CalciteExplainQueryTest.java | 47 ++-- .../sql/calcite/CalciteInsertDmlTest.java | 50 +++- .../sql/calcite/CalciteReplaceDmlTest.java | 47 +++- .../sql/calcite/CalciteSelectQueryTest.java | 22 +- .../sql/calcite/IngestTableFunctionTest.java | 3 +- .../druid/sql/http/SqlResourceTest.java | 4 +- 16 files changed, 388 insertions(+), 151 deletions(-) create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/planner/ExplainAttributes.java diff --git a/docs/querying/sql-translation.md b/docs/querying/sql-translation.md index 370ad002c1c..39014f1b28d 100644 --- a/docs/querying/sql-translation.md +++ b/docs/querying/sql-translation.md @@ -64,7 +64,11 @@ appreciated. The [EXPLAIN PLAN](sql.md#explain-plan) functionality can help you understand how a given SQL query will be translated to native. -EXPLAIN PLAN statements return a `RESOURCES` column that describes the resource being queried as well as a `PLAN` column that contains a JSON array of native queries that Druid will run. +EXPLAIN PLAN statements return: +- a `PLAN` column that contains a JSON array of native queries that Druid will run +- a `RESOURCES` column that describes the resource being queried as well as a `PLAN` column that contains a JSON array of native queries that Druid will run +- a `ATTRIBUTES` column that describes the attributes of a query, such as the statement type and target data source + For example, consider the following query: ```sql @@ -77,120 +81,132 @@ WHERE channel IN (SELECT page FROM wikipedia GROUP BY page ORDER BY COUNT(*) DES GROUP BY channel ``` -The EXPLAIN PLAN statement returns the following plan: +The EXPLAIN PLAN statement returns the following result with plan, resources, and attributes information in it: ```json [ - { - "query": { - "queryType": "topN", - "dataSource": { - "type": "join", - "left": { - "type": "table", - "name": "wikipedia" - }, - "right": { - "type": "query", - "query": { - "queryType": "groupBy", - "dataSource": { - "type": "table", - "name": "wikipedia" - }, - "intervals": { - "type": "intervals", - "intervals": [ - "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" - ] - }, - "granularity": { - "type": "all" - }, - "dimensions": [ - { - "type": "default", - "dimension": "page", - "outputName": "d0", - "outputType": "STRING" - } - ], - "aggregations": [ - { - "type": "count", - "name": "a0" - } - ], - "limitSpec": { - "type": "default", - "columns": [ + [ + { + "query": { + "queryType": "topN", + "dataSource": { + "type": "join", + "left": { + "type": "table", + "name": "wikipedia" + }, + "right": { + "type": "query", + "query": { + "queryType": "groupBy", + "dataSource": { + "type": "table", + "name": "wikipedia" + }, + "intervals": { + "type": "intervals", + "intervals": [ + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" + ] + }, + "granularity": { + "type": "all" + }, + "dimensions": [ { - "dimension": "a0", - "direction": "descending", - "dimensionOrder": { - "type": "numeric" - } + "type": "default", + "dimension": "page", + "outputName": "d0", + "outputType": "STRING" } ], - "limit": 10 - }, - "context": { - "sqlOuterLimit": 101, - "sqlQueryId": "ee616a36-c30c-4eae-af00-245127956e42", - "useApproximateCountDistinct": false, - "useApproximateTopN": false + "aggregations": [ + { + "type": "count", + "name": "a0" + } + ], + "limitSpec": { + "type": "default", + "columns": [ + { + "dimension": "a0", + "direction": "descending", + "dimensionOrder": { + "type": "numeric" + } + } + ], + "limit": 10 + }, + "context": { + "sqlOuterLimit": 101, + "sqlQueryId": "ee616a36-c30c-4eae-af00-245127956e42", + "useApproximateCountDistinct": false, + "useApproximateTopN": false + } } + }, + "rightPrefix": "j0.", + "condition": "(\"channel\" == \"j0.d0\")", + "joinType": "INNER" + }, + "dimension": { + "type": "default", + "dimension": "channel", + "outputName": "d0", + "outputType": "STRING" + }, + "metric": { + "type": "dimension", + "ordering": { + "type": "lexicographic" } }, - "rightPrefix": "j0.", - "condition": "(\"channel\" == \"j0.d0\")", - "joinType": "INNER" - }, - "dimension": { - "type": "default", - "dimension": "channel", - "outputName": "d0", - "outputType": "STRING" - }, - "metric": { - "type": "dimension", - "ordering": { - "type": "lexicographic" + "threshold": 101, + "intervals": { + "type": "intervals", + "intervals": [ + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" + ] + }, + "granularity": { + "type": "all" + }, + "aggregations": [ + { + "type": "count", + "name": "a0" + } + ], + "context": { + "sqlOuterLimit": 101, + "sqlQueryId": "ee616a36-c30c-4eae-af00-245127956e42", + "useApproximateCountDistinct": false, + "useApproximateTopN": false } }, - "threshold": 101, - "intervals": { - "type": "intervals", - "intervals": [ - "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" - ] - }, - "granularity": { - "type": "all" - }, - "aggregations": [ + "signature": [ { - "type": "count", - "name": "a0" + "name": "d0", + "type": "STRING" + }, + { + "name": "a0", + "type": "LONG" } - ], - "context": { - "sqlOuterLimit": 101, - "sqlQueryId": "ee616a36-c30c-4eae-af00-245127956e42", - "useApproximateCountDistinct": false, - "useApproximateTopN": false - } - }, - "signature": [ - { - "name": "d0", - "type": "STRING" - }, - { - "name": "a0", - "type": "LONG" - } - ] + ] + } + ], + [ + { + "name": "wikipedia", + "type": "DATASOURCE" + } + ], + { + "statementType": "SELECT", + "targetDataSource": null } ] ``` diff --git a/docs/querying/sql.md b/docs/querying/sql.md index 18c3b84272f..58889896128 100644 --- a/docs/querying/sql.md +++ b/docs/querying/sql.md @@ -250,8 +250,8 @@ Add "EXPLAIN PLAN FOR" to the beginning of any query to get information about ho the query will not actually be executed. Refer to the [Query translation](sql-translation.md#interpreting-explain-plan-output) documentation for more information on the output of EXPLAIN PLAN. -> Be careful when interpreting EXPLAIN PLAN output, and use [request logging](../configuration/index.md#request-logging) if in doubt. -Request logs show the exact native query that will be run. +> For the legacy plan, be careful when interpreting EXPLAIN PLAN output, and use [request logging](../configuration/index.md#request-logging) if in doubt. +Request logs show the exact native query that will be run. Alternatively, to see the native query plan, set `useNativeQueryExplain` to true in the query context. ## Identifiers and literals diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java index b0912507a27..59c4cca2851 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java @@ -135,6 +135,7 @@ public class DruidPlanner implements Closeable try { handler.validate(); plannerContext.setResourceActions(handler.resourceActions()); + plannerContext.setExplainAttributes(handler.explainAttributes()); } catch (RuntimeException e) { throw new ValidationException(e); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/ExplainAttributes.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ExplainAttributes.java new file mode 100644 index 00000000000..b793fad96fc --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ExplainAttributes.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.planner; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.calcite.sql.SqlNode; + +import javax.annotation.Nullable; + +/** + * ExplainAttributes holds the attributes of a SQL statement that is used in the EXPLAIN PLAN result. + */ +public final class ExplainAttributes +{ + private final String statementType; + + @Nullable + private final SqlNode targetDataSource; + + public ExplainAttributes( + @JsonProperty("statementType") final String statementType, + @JsonProperty("targetDataSource") @Nullable final SqlNode targetDataSource) + { + this.statementType = statementType; + this.targetDataSource = targetDataSource; + } + + /** + * @return the statement kind of a SQL statement. For example, SELECT, INSERT, or REPLACE. + */ + @JsonProperty + public String getStatementType() + { + return statementType; + } + + /** + * @return the target datasource in a SQL statement. Returns null + * for SELECT/non-DML statements where there is no target datasource. + */ + @Nullable + @JsonProperty + public String getTargetDataSource() + { + return targetDataSource == null ? null : targetDataSource.toString(); + } + + @Override + public String toString() + { + return "ExplainAttributes{" + + "statementType='" + statementType + '\'' + + ", targetDataSource=" + targetDataSource + + '}'; + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/IngestHandler.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/IngestHandler.java index 47ea8556038..7aab7148b4e 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/IngestHandler.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/IngestHandler.java @@ -273,6 +273,15 @@ public abstract class IngestHandler extends QueryHandler } super.validate(); } + + @Override + public ExplainAttributes explainAttributes() + { + return new ExplainAttributes( + DruidSqlInsert.OPERATOR.getName(), + sqlNode.getTargetTable() + ); + } } /** @@ -331,5 +340,14 @@ public abstract class IngestHandler extends QueryHandler ); } } + + @Override + public ExplainAttributes explainAttributes() + { + return new ExplainAttributes( + DruidSqlReplace.OPERATOR.getName(), + sqlNode.getTargetTable() + ); + } } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java index 280372462a3..5c36ef72b66 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java @@ -111,6 +111,8 @@ public class PlannerContext private String planningError; private QueryMaker queryMaker; private VirtualColumnRegistry joinExpressionVirtualColumnRegistry; + // set of attributes for a SQL statement used in the EXPLAIN PLAN output + private ExplainAttributes explainAttributes; private PlannerContext( final PlannerToolbox plannerToolbox, @@ -502,4 +504,18 @@ public class PlannerContext { this.joinExpressionVirtualColumnRegistry = joinExpressionVirtualColumnRegistry; } + + public ExplainAttributes getExplainAttributes() + { + return this.explainAttributes; + } + + public void setExplainAttributes(ExplainAttributes explainAttributes) + { + if (this.explainAttributes != null) { + throw new ISE("ExplainAttributes has already been set"); + } + this.explainAttributes = explainAttributes; + } + } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/QueryHandler.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/QueryHandler.java index e5808e97375..4e9cc3e46bb 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/QueryHandler.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/QueryHandler.java @@ -173,10 +173,11 @@ public abstract class QueryHandler extends SqlStatementHandler.BaseStatementHand { return typeFactory.createStructType( ImmutableList.of( + Calcites.createSqlType(typeFactory, SqlTypeName.VARCHAR), Calcites.createSqlType(typeFactory, SqlTypeName.VARCHAR), Calcites.createSqlType(typeFactory, SqlTypeName.VARCHAR) ), - ImmutableList.of("PLAN", "RESOURCES") + ImmutableList.of("PLAN", "RESOURCES", "ATTRIBUTES") ); } @@ -229,6 +230,15 @@ public abstract class QueryHandler extends SqlStatementHandler.BaseStatementHand } } + @Override + public ExplainAttributes explainAttributes() + { + return new ExplainAttributes( + "SELECT", + null + ); + } + private static Set getBindableTables(final RelNode relNode) { class HasBindableVisitor extends RelVisitor @@ -374,9 +384,19 @@ public abstract class QueryHandler extends SqlStatementHandler.BaseStatementHand log.error(jpe, "Encountered exception while serializing resources for explain output"); resourcesString = null; } + + String explainAttributesString; + try { + explainAttributesString = plannerContext.getJsonMapper().writeValueAsString(plannerContext.getExplainAttributes()); + } + catch (JsonProcessingException jpe) { + log.error(jpe, "Encountered exception while serializing attributes for explain output"); + explainAttributesString = null; + } + final Supplier> resultsSupplier = Suppliers.ofInstance( QueryResponse.withEmptyContext( - Sequences.simple(ImmutableList.of(new Object[]{explanation, resourcesString})) + Sequences.simple(ImmutableList.of(new Object[]{explanation, resourcesString, explainAttributesString})) ) ); return new PlannerResult(resultsSupplier, getExplainStructType(rel.getCluster().getTypeFactory())); @@ -384,7 +404,7 @@ public abstract class QueryHandler extends SqlStatementHandler.BaseStatementHand /** * This method doesn't utilize the Calcite's internal {@link RelOptUtil#dumpPlan} since that tends to be verbose - * and not indicative of the native Druid Queries which will get executed + * and not indicative of the native Druid Queries which will get executed. * This method assumes that the Planner has converted the RelNodes to DruidRels, and thereby we can implicitly cast it * * @param rel Instance of the root {@link DruidRel} which is formed by running the planner transformations on it diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/SqlStatementHandler.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/SqlStatementHandler.java index 70847516902..4cb263c5220 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/SqlStatementHandler.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/SqlStatementHandler.java @@ -41,6 +41,7 @@ public interface SqlStatementHandler void prepare(); PrepareResult prepareResult(); PlannerResult plan() throws ValidationException; + ExplainAttributes explainAttributes(); /** * Context available to statement handlers. diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java index 51f536f41ac..6de7316729b 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java @@ -469,7 +469,9 @@ public class DruidAvaticaHandlerTest extends CalciteTestBase DUMMY_SQL_QUERY_ID ), "RESOURCES", - "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]" + "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]", + "ATTRIBUTES", + "{\"statementType\":\"SELECT\",\"targetDataSource\":null}" ) ), getRows(resultSet) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index 0bc2dc3ffc4..2a0930912b4 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -170,6 +170,9 @@ public class BaseCalciteQueryTest extends CalciteTestBase public static final PlannerConfig PLANNER_CONFIG_AUTHORIZE_SYS_TABLES = PlannerConfig.builder().authorizeSystemTablesDirectly(true).build(); + public static final PlannerConfig PLANNER_CONFIG_LEGACY_QUERY_EXPLAIN = + PlannerConfig.builder().useNativeQueryExplain(false).build(); + public static final PlannerConfig PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN = PlannerConfig.builder().useNativeQueryExplain(true).build(); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java index 480fca6aeb8..0c694e372c1 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java @@ -51,14 +51,15 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest + "\"signature\":[{\"name\":\"a0\",\"type\":\"LONG\"}]" + "}]"; final String resources = "[{\"name\":\"aview\",\"type\":\"VIEW\"}]"; + final String attributes = "{\"statementType\":\"SELECT\",\"targetDataSource\":null}"; testQuery( - PlannerConfig.builder().useNativeQueryExplain(false).build(), + PLANNER_CONFIG_LEGACY_QUERY_EXPLAIN, query, CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), ImmutableList.of( - new Object[]{legacyExplanation, resources} + new Object[]{legacyExplanation, resources, attributes} ) ); testQuery( @@ -67,7 +68,7 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), ImmutableList.of( - new Object[]{explanation, resources} + new Object[]{explanation, resources, attributes} ) ); } @@ -81,6 +82,7 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest + " BindableTableScan(table=[[INFORMATION_SCHEMA, COLUMNS]])\n"; final String resources = "[]"; + final String attributes = "{\"statementType\":\"SELECT\",\"targetDataSource\":null}"; testQuery( "EXPLAIN PLAN FOR\n" @@ -89,7 +91,7 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest + "WHERE TABLE_SCHEMA = 'druid' AND TABLE_NAME = 'foo'", ImmutableList.of(), ImmutableList.of( - new Object[]{explanation, resources} + new Object[]{explanation, resources, attributes} ) ); } @@ -125,23 +127,24 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest + "\"signature\":[{\"name\":\"a0\",\"type\":\"LONG\"}]" + "}]"; final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; + final String attributes = "{\"statementType\":\"SELECT\",\"targetDataSource\":null}"; testQuery( query, ImmutableList.of(), - ImmutableList.of(new Object[]{explanation, resources}) + ImmutableList.of(new Object[]{explanation, resources, attributes}) ); testQuery( - PlannerConfig.builder().useNativeQueryExplain(false).build(), + PLANNER_CONFIG_LEGACY_QUERY_EXPLAIN, query, CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), - ImmutableList.of(new Object[]{legacyExplanation, resources}) + ImmutableList.of(new Object[]{legacyExplanation, resources, attributes}) ); } - // This testcase has been added here and not in CalciteSelectQueryTests since this checks if the overrides are working + // This testcase has been added here and not in CalciteSelectQueryTest since this checks if the overrides are working // properly when displaying the output of "EXPLAIN PLAN FOR ..." queries @Test public void testExplainSelectStarWithOverrides() @@ -181,16 +184,17 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest + "}]"; String sql = "EXPLAIN PLAN FOR SELECT * FROM druid.foo"; String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; + final String attributes = "{\"statementType\":\"SELECT\",\"targetDataSource\":null}"; // Test when default config and no overrides - testQuery(sql, ImmutableList.of(), ImmutableList.of(new Object[]{explanation, resources})); + testQuery(sql, ImmutableList.of(), ImmutableList.of(new Object[]{explanation, resources, attributes})); // Test when default config and useNativeQueryExplain is overridden in the context testQuery( sql, legacyExplainContext, ImmutableList.of(), - ImmutableList.of(new Object[]{legacyExplanationWithContext, resources}) + ImmutableList.of(new Object[]{legacyExplanationWithContext, resources, attributes}) ); // Test when useNativeQueryExplain enabled by default and no overrides @@ -199,7 +203,7 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest sql, CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), - ImmutableList.of(new Object[]{explanation, resources}) + ImmutableList.of(new Object[]{explanation, resources, attributes}) ); // Test when useNativeQueryExplain enabled by default but is overriden in the context @@ -209,7 +213,7 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest sql, CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), - ImmutableList.of(new Object[]{explanationWithContext, resources}) + ImmutableList.of(new Object[]{explanationWithContext, resources, attributes}) ); } @@ -241,14 +245,14 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest + "\"signature\":[{\"name\":\"dim1\",\"type\":\"STRING\"}]" + "}]"; final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; - + final String attributes = "{\"statementType\":\"SELECT\",\"targetDataSource\":null}"; testQuery( - PlannerConfig.builder().useNativeQueryExplain(false).build(), + PLANNER_CONFIG_LEGACY_QUERY_EXPLAIN, query, CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), ImmutableList.of( - new Object[]{legacyExplanation, resources} + new Object[]{legacyExplanation, resources, attributes} ) ); testQuery( @@ -257,7 +261,7 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), ImmutableList.of( - new Object[]{explanation, resources} + new Object[]{explanation, resources, attributes} ) ); } @@ -294,12 +298,12 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest + "\"signature\":[{\"name\":\"v0\",\"type\":\"STRING\"},{\"name\":\"v1\",\"type\":\"STRING\"}]" + "}]"; final String expectedResources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; - + final String expectedAttributes = "{\"statementType\":\"SELECT\",\"targetDataSource\":null}"; testQuery( explainSql, defaultExprContext, ImmutableList.of(), - ImmutableList.of(new Object[]{expectedPlanWithDefaultExpressions, expectedResources}) + ImmutableList.of(new Object[]{expectedPlanWithDefaultExpressions, expectedResources, expectedAttributes}) ); // Test plan as mv-filtered virtual columns @@ -318,7 +322,6 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest + "\"granularity\":{\"type\":\"all\"}}," + "\"signature\":[{\"name\":\"v0\",\"type\":\"STRING\"},{\"name\":\"v1\",\"type\":\"STRING\"}]" + "}]"; - final Map mvFilteredContext = new HashMap<>(QUERY_CONTEXT_DEFAULT); mvFilteredContext.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN, true); @@ -326,7 +329,7 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest explainSql, mvFilteredContext, ImmutableList.of(), - ImmutableList.of(new Object[]{expectedPlanWithMvfiltered, expectedResources}) + ImmutableList.of(new Object[]{expectedPlanWithMvfiltered, expectedResources, expectedAttributes}) ); } @@ -358,13 +361,13 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest + "\"signature\":[{\"name\":\"v0\",\"type\":\"LONG\"}]" + "}]"; final String expectedResources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; - + final String expectedAttributes = "{\"statementType\":\"SELECT\",\"targetDataSource\":null}"; // Verify the query plan testQuery( explainSql, queryContext, ImmutableList.of(), - ImmutableList.of(new Object[]{expectedPlan, expectedResources}) + ImmutableList.of(new Object[]{expectedPlan, expectedResources, expectedAttributes}) ); } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java index fb184e89c88..c8184d97066 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java @@ -48,7 +48,6 @@ import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.parser.DruidSqlInsert; import org.apache.druid.sql.calcite.planner.Calcites; import org.apache.druid.sql.calcite.planner.IngestHandler; -import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.util.CalciteTests; import org.hamcrest.CoreMatchers; @@ -870,14 +869,30 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest ) .build(); - final String expectedExplanation = + final String legacyExplanation = "DruidQueryRel(query=[" + queryJsonMapper.writeValueAsString(expectedQuery) + "], signature=[{x:STRING, y:STRING, z:LONG}])\n"; + final String explanation = + "[" + + "{\"query\":{\"queryType\":\"scan\"," + + "\"dataSource\":{\"type\":\"external\",\"inputSource\":{\"type\":\"inline\",\"data\":\"a,b,1\\nc,d,2\\n\"}," + + "\"inputFormat\":{\"type\":\"csv\",\"columns\":[\"x\",\"y\",\"z\"]}," + + "\"signature\":[{\"name\":\"x\",\"type\":\"STRING\"},{\"name\":\"y\",\"type\":\"STRING\"},{\"name\":\"z\",\"type\":\"LONG\"}]}," + + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + + "\"resultFormat\":\"compactedList\",\"columns\":[\"x\",\"y\",\"z\"],\"legacy\":false," + + "\"context\":{\"sqlInsertSegmentGranularity\":\"{\\\"type\\\":\\\"all\\\"}\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + + "\"granularity\":{\"type\":\"all\"}},\"signature\":[{\"name\":\"x\",\"type\":\"STRING\"},{\"name\":\"y\",\"type\":\"STRING\"},{\"name\":\"z\",\"type\":\"LONG\"}]" + + "}]"; + + final String resources = "[{\"name\":\"EXTERNAL\",\"type\":\"EXTERNAL\"},{\"name\":\"dst\",\"type\":\"DATASOURCE\"}]"; + final String attributes = "{\"statementType\":\"INSERT\",\"targetDataSource\":\"dst\"}"; + + // Use testQuery for EXPLAIN (not testIngestionQuery). testQuery( - PlannerConfig.builder().useNativeQueryExplain(false).build(), + PLANNER_CONFIG_LEGACY_QUERY_EXPLAIN, ImmutableMap.of("sqlQueryId", "dummy"), Collections.emptyList(), StringUtils.format( @@ -889,8 +904,33 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest new DefaultResultsVerifier( ImmutableList.of( new Object[]{ - expectedExplanation, - "[{\"name\":\"EXTERNAL\",\"type\":\"EXTERNAL\"},{\"name\":\"dst\",\"type\":\"DATASOURCE\"}]" + legacyExplanation, + resources, + attributes + } + ), + null + ), + null + ); + + + testQuery( + PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN, + ImmutableMap.of("sqlQueryId", "dummy"), + Collections.emptyList(), + StringUtils.format( + "EXPLAIN PLAN FOR INSERT INTO dst SELECT * FROM %s PARTITIONED BY ALL TIME", + externSql(externalDataSource) + ), + CalciteTests.SUPER_USER_AUTH_RESULT, + ImmutableList.of(), + new DefaultResultsVerifier( + ImmutableList.of( + new Object[]{ + explanation, + resources, + attributes } ), null diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteReplaceDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteReplaceDmlTest.java index c51aac37c07..e78485416cd 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteReplaceDmlTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteReplaceDmlTest.java @@ -39,7 +39,6 @@ import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.parser.DruidSqlInsert; import org.apache.druid.sql.calcite.parser.DruidSqlParserUtils; import org.apache.druid.sql.calcite.parser.DruidSqlReplace; -import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.util.CalciteTests; import org.junit.Assert; @@ -614,14 +613,28 @@ public class CalciteReplaceDmlTest extends CalciteIngestionDmlTest ) .build(); - final String expectedExplanation = + final String legacyExplanation = "DruidQueryRel(query=[" + queryJsonMapper.writeValueAsString(expectedQuery) + "], signature=[{x:STRING, y:STRING, z:LONG}])\n"; + final String explanation = "[{" + + "\"query\":{\"queryType\":\"scan\"," + + "\"dataSource\":{\"type\":\"external\",\"inputSource\":{\"type\":\"inline\",\"data\":\"a,b,1\\nc,d,2\\n\"}," + + "\"inputFormat\":{\"type\":\"csv\",\"columns\":[\"x\",\"y\",\"z\"]}," + + "\"signature\":[{\"name\":\"x\",\"type\":\"STRING\"},{\"name\":\"y\",\"type\":\"STRING\"},{\"name\":\"z\",\"type\":\"LONG\"}]}," + + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + + "\"resultFormat\":\"compactedList\",\"columns\":[\"x\",\"y\",\"z\"],\"legacy\":false," + + "\"context\":{\"sqlInsertSegmentGranularity\":\"{\\\"type\\\":\\\"all\\\"}\",\"sqlQueryId\":\"dummy\"," + + "\"sqlReplaceTimeChunks\":\"all\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"granularity\":{\"type\":\"all\"}}," + + "\"signature\":[{\"name\":\"x\",\"type\":\"STRING\"},{\"name\":\"y\",\"type\":\"STRING\"},{\"name\":\"z\",\"type\":\"LONG\"}]}]"; + + final String resources = "[{\"name\":\"EXTERNAL\",\"type\":\"EXTERNAL\"},{\"name\":\"dst\",\"type\":\"DATASOURCE\"}]"; + final String attributes = "{\"statementType\":\"REPLACE\",\"targetDataSource\":\"dst\"}"; + // Use testQuery for EXPLAIN (not testIngestionQuery). testQuery( - PlannerConfig.builder().useNativeQueryExplain(false).build(), + PLANNER_CONFIG_LEGACY_QUERY_EXPLAIN, ImmutableMap.of("sqlQueryId", "dummy"), Collections.emptyList(), StringUtils.format( @@ -633,8 +646,32 @@ public class CalciteReplaceDmlTest extends CalciteIngestionDmlTest new DefaultResultsVerifier( ImmutableList.of( new Object[]{ - expectedExplanation, - "[{\"name\":\"EXTERNAL\",\"type\":\"EXTERNAL\"},{\"name\":\"dst\",\"type\":\"DATASOURCE\"}]" + legacyExplanation, + resources, + attributes + } + ), + null + ), + null + ); + + testQuery( + PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN, + ImmutableMap.of("sqlQueryId", "dummy"), + Collections.emptyList(), + StringUtils.format( + "EXPLAIN PLAN FOR REPLACE INTO dst OVERWRITE ALL SELECT * FROM %s PARTITIONED BY ALL TIME", + externSql(externalDataSource) + ), + CalciteTests.SUPER_USER_AUTH_RESULT, + ImmutableList.of(), + new DefaultResultsVerifier( + ImmutableList.of( + new Object[]{ + explanation, + resources, + attributes } ), null diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java index 0982f17807c..f8e31d7e922 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java @@ -45,7 +45,6 @@ import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.sql.SqlPlanningException; import org.apache.druid.sql.calcite.filtration.Filtration; -import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.util.CalciteTests; import org.joda.time.DateTime; @@ -544,16 +543,18 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest + "}]"; final String legacyExplanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"EXPR$0\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"granularity\":{\"type\":\"all\"}}], signature=[{EXPR$0:LONG}])\n"; final String resources = "[]"; + final String attributes = "{\"statementType\":\"SELECT\",\"targetDataSource\":null}"; testQuery( - PlannerConfig.builder().useNativeQueryExplain(false).build(), + PLANNER_CONFIG_LEGACY_QUERY_EXPLAIN, query, CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), ImmutableList.of( new Object[]{ legacyExplanation, - resources + resources, + attributes } ) ); @@ -565,7 +566,8 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest ImmutableList.of( new Object[]{ explanation, - resources + resources, + attributes } ) ); @@ -1284,19 +1286,20 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest + "\"legacy\":false," + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + "\"granularity\":{\"type\":\"all\"}}," - + "\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX\"}]" - + "}]"; + + "\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX\"}]}]"; final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; + final String attributes = "{\"statementType\":\"SELECT\",\"targetDataSource\":null}"; testQuery( - PlannerConfig.builder().useNativeQueryExplain(false).build(), + PLANNER_CONFIG_LEGACY_QUERY_EXPLAIN, query, CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), ImmutableList.of( new Object[]{ legacyExplanation, - resources + resources, + attributes } ) ); @@ -1308,7 +1311,8 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest ImmutableList.of( new Object[]{ explanation, - resources + resources, + attributes } ) ); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/IngestTableFunctionTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/IngestTableFunctionTest.java index 86136c70002..825198eefbd 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/IngestTableFunctionTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/IngestTableFunctionTest.java @@ -317,6 +317,7 @@ public class IngestTableFunctionTest extends CalciteIngestionDmlTest "\"granularity\":{\"type\":\"all\"}}," + "\"signature\":[{\"name\":\"x\",\"type\":\"STRING\"},{\"name\":\"y\",\"type\":\"STRING\"},{\"name\":\"z\",\"type\":\"LONG\"}]}]"; final String resources = "[{\"name\":\"EXTERNAL\",\"type\":\"EXTERNAL\"},{\"name\":\"dst\",\"type\":\"DATASOURCE\"}]"; + final String attributes = "{\"statementType\":\"INSERT\",\"targetDataSource\":\"dst\"}"; testQuery( PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN, @@ -324,7 +325,7 @@ public class IngestTableFunctionTest extends CalciteIngestionDmlTest CalciteTests.SUPER_USER_AUTH_RESULT, ImmutableList.of(), ImmutableList.of( - new Object[]{explanation, resources} + new Object[]{explanation, resources, attributes} ) ); didTest = true; diff --git a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java index 339e713e309..2a80011a9b0 100644 --- a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java +++ b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java @@ -1326,7 +1326,9 @@ public class SqlResourceTest extends CalciteTestBase "false" ), "RESOURCES", - "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]" + "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]", + "ATTRIBUTES", + "{\"statementType\":\"SELECT\",\"targetDataSource\":null}" ) ), rows