From f7c63169924611969139783a711275f33f694676 Mon Sep 17 00:00:00 2001 From: Karan Kumar Date: Wed, 24 Aug 2022 17:39:55 +0530 Subject: [PATCH] Setting useNativeQueryExplain to true (#12936) * Setting useNativeQueryExplain to true * Update docs/querying/sql-query-context.md Co-authored-by: Santosh Pingale * Fixing tests * Fixing broken tests Co-authored-by: Santosh Pingale --- docs/configuration/index.md | 2 +- docs/querying/sql-query-context.md | 2 +- .../sql/calcite/planner/PlannerConfig.java | 2 +- .../sql/avatica/DruidAvaticaHandlerTest.java | 5 ++-- .../sql/calcite/CalciteExplainQueryTest.java | 25 +++++++++++-------- .../sql/calcite/CalciteInsertDmlTest.java | 2 +- .../sql/calcite/CalciteReplaceDmlTest.java | 2 +- .../sql/calcite/CalciteSelectQueryTest.java | 5 ++++ .../druid/sql/http/SqlResourceTest.java | 14 ++++++++--- 9 files changed, 37 insertions(+), 22 deletions(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 5e6f7d7e865..06939316b84 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -1878,7 +1878,7 @@ The Druid SQL server is configured through the following properties on the Broke |`druid.sql.planner.metadataSegmentCacheEnable`|Whether to keep a cache of published segments in broker. If true, broker polls coordinator in background to get segments from metadata store and maintains a local cache. If false, coordinator's REST API will be invoked when broker needs published segments info.|false| |`druid.sql.planner.metadataSegmentPollPeriod`|How often to poll coordinator for published segments list if `druid.sql.planner.metadataSegmentCacheEnable` is set to true. Poll period is in milliseconds. |60000| |`druid.sql.planner.authorizeSystemTablesDirectly`|If true, Druid authorizes queries against any of the system schema tables (`sys` in SQL) as `SYSTEM_TABLE` resources which require `READ` access, in addition to permissions based content filtering.|false| -|`druid.sql.planner.useNativeQueryExplain`|If true, `EXPLAIN PLAN FOR` will return the explain plan as a JSON representation of equivalent native query(s), else it will return the original version of explain plan generated by Calcite. It can be overridden per query with `useNativeQueryExplain` context key.|false| +|`druid.sql.planner.useNativeQueryExplain`|If true, `EXPLAIN PLAN FOR` will return the explain plan as a JSON representation of equivalent native query(s), else it will return the original version of explain plan generated by Calcite. It can be overridden per query with `useNativeQueryExplain` context key.|true| |`druid.sql.planner.maxNumericInFilters`|Max limit for the amount of numeric values that can be compared for a string type dimension when the entire SQL WHERE clause of a query translates to an [OR](../querying/filters.md#or) of [Bound filter](../querying/filters.md#bound-filter). By default, Druid does not restrict the amount of numeric Bound Filters on String columns, although this situation may block other queries from running. Set this property to a smaller value to prevent Druid from running queries that have prohibitively long segment processing times. The optimal limit requires some trial and error; we recommend starting with 100. Users who submit a query that exceeds the limit of `maxNumericInFilters` should instead rewrite their queries to use strings in the `WHERE` clause instead of numbers. For example, `WHERE someString IN (‘123’, ‘456’)`. If this value is disabled, `maxNumericInFilters` set through query context is ignored.|`-1` (disabled)| |`druid.sql.approxCountDistinct.function`|Implementation to use for the [`APPROX_COUNT_DISTINCT` function](../querying/sql-aggregations.md). Without extensions loaded, the only valid value is `APPROX_COUNT_DISTINCT_BUILTIN` (a HyperLogLog, or HLL, based implementation). If the [DataSketches extension](../development/extensions-core/datasketches-extension.md) is loaded, this can also be `APPROX_COUNT_DISTINCT_DS_HLL` (alternative HLL implementation) or `APPROX_COUNT_DISTINCT_DS_THETA`.

Theta sketches use significantly more memory than HLL sketches, so you should prefer one of the two HLL implementations.|APPROX_COUNT_DISTINCT_BUILTIN| diff --git a/docs/querying/sql-query-context.md b/docs/querying/sql-query-context.md index 6b7a746623e..550ce3c78c9 100644 --- a/docs/querying/sql-query-context.md +++ b/docs/querying/sql-query-context.md @@ -42,7 +42,7 @@ Configure Druid SQL query planning using the parameters in the table below. |`useGroupingSetForExactDistinct`|Whether to use grouping sets to execute queries with multiple exact distinct aggregations.|druid.sql.planner.useGroupingSetForExactDistinct on the Broker (default: false)| |`useApproximateTopN`|Whether to use approximate [TopN queries](topnquery.md) when a SQL query could be expressed as such. If false, exact [GroupBy queries](groupbyquery.md) will be used instead.|druid.sql.planner.useApproximateTopN on the Broker (default: true)| |`enableTimeBoundaryPlanning`|If true, SQL queries will get converted to TimeBoundary queries wherever possible. TimeBoundary queries are very efficient for min-max calculation on __time column in a datasource |druid.query.default.context.enableTimeBoundaryPlanning on the Broker (default: false)| -|`useNativeQueryExplain`|If true, `EXPLAIN PLAN FOR` will return the explain plan as a JSON representation of equivalent native query(s), else it will return the original version of explain plan generated by Calcite.|druid.sql.planner.useNativeQueryExplain on the Broker (default: False)| +|`useNativeQueryExplain`|If true, `EXPLAIN PLAN FOR` will return the explain plan as a JSON representation of equivalent native query(s), else it will return the original version of explain plan generated by Calcite.|`druid.sql.planner.useNativeQueryExplain` on the Broker (default: true)| ## Setting the query context The query context parameters can be specified as a "context" object in the [JSON API](sql-api.md) or as a [JDBC connection properties object](sql-jdbc.md). diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java index b450f777694..21fe70799bb 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java @@ -75,7 +75,7 @@ public class PlannerConfig private boolean authorizeSystemTablesDirectly = false; @JsonProperty - private boolean useNativeQueryExplain = false; + private boolean useNativeQueryExplain = true; @JsonProperty private boolean forceExpressionVirtualColumns = false; diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java index 850fa362318..03763c4b5e8 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java @@ -397,8 +397,9 @@ public class DruidAvaticaHandlerTest extends CalciteTestBase ImmutableList.of( ImmutableMap.of( "PLAN", - StringUtils.format("DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"context\":{\"sqlQueryId\":\"%s\",\"sqlStringifyArrays\":false,\"sqlTimeZone\":\"America/Los_Angeles\"}}], signature=[{a0:LONG}])\n", - DUMMY_SQL_QUERY_ID + StringUtils.format( + "[{\"query\":{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"context\":{\"sqlQueryId\":\"%s\",\"sqlStringifyArrays\":false,\"sqlTimeZone\":\"America/Los_Angeles\"}},\"signature\":[{\"name\":\"a0\",\"type\":\"LONG\"}]}]", + DUMMY_SQL_QUERY_ID ), "RESOURCES", "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]" diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java index daa63f3f49f..70e0320e3a1 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java @@ -53,7 +53,9 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest final String resources = "[{\"name\":\"aview\",\"type\":\"VIEW\"}]"; testQuery( + PlannerConfig.builder().useNativeQueryExplain(false).build(), query, + CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), ImmutableList.of( new Object[]{legacyExplanation, resources} @@ -127,15 +129,15 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest testQuery( query, ImmutableList.of(), - ImmutableList.of(new Object[]{legacyExplanation, resources}) + ImmutableList.of(new Object[]{explanation, resources}) ); testQuery( - PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN, + PlannerConfig.builder().useNativeQueryExplain(false).build(), query, CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), - ImmutableList.of(new Object[]{explanation, resources}) + ImmutableList.of(new Object[]{legacyExplanation, resources}) ); } @@ -145,15 +147,14 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest public void testExplainSelectStarWithOverrides() { Map useRegularExplainContext = new HashMap<>(QUERY_CONTEXT_DEFAULT); - useRegularExplainContext.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN, false); + useRegularExplainContext.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN, true); - Map useNativeQueryExplain = new HashMap<>(QUERY_CONTEXT_DEFAULT); - useNativeQueryExplain.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN, true); + Map legacyExplainContext = new HashMap<>(QUERY_CONTEXT_DEFAULT); + legacyExplainContext.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN, false); // Skip vectorization since otherwise the "context" will change for each subtest. skipVectorize(); - String legacyExplanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, dim1:STRING, dim2:STRING, dim3:STRING, cnt:LONG, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n"; String legacyExplanationWithContext = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"useNativeQueryExplain\":false,\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, dim1:STRING, dim2:STRING, dim3:STRING, cnt:LONG, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n"; String explanation = "[{" + "\"query\":{\"queryType\":\"scan\"," @@ -182,14 +183,14 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; // Test when default config and no overrides - testQuery(sql, ImmutableList.of(), ImmutableList.of(new Object[]{legacyExplanation, resources})); + testQuery(sql, ImmutableList.of(), ImmutableList.of(new Object[]{explanation, resources})); // Test when default config and useNativeQueryExplain is overridden in the context testQuery( sql, - useNativeQueryExplain, + legacyExplainContext, ImmutableList.of(), - ImmutableList.of(new Object[]{explanationWithContext, resources}) + ImmutableList.of(new Object[]{legacyExplanationWithContext, resources}) ); // Test when useNativeQueryExplain enabled by default and no overrides @@ -208,7 +209,7 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest sql, CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), - ImmutableList.of(new Object[]{legacyExplanationWithContext, resources}) + ImmutableList.of(new Object[]{explanationWithContext, resources}) ); } @@ -242,7 +243,9 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; testQuery( + PlannerConfig.builder().useNativeQueryExplain(false).build(), query, + CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), ImmutableList.of( new Object[]{legacyExplanation, resources} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java index fccbf1682a4..92d0a5773f0 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java @@ -611,7 +611,7 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest // Use testQuery for EXPLAIN (not testIngestionQuery). testQuery( - new PlannerConfig(), + PlannerConfig.builder().useNativeQueryExplain(false).build(), ImmutableMap.of("sqlQueryId", "dummy"), Collections.emptyList(), StringUtils.format( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteReplaceDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteReplaceDmlTest.java index 2cbd53b54aa..a244abef6ab 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteReplaceDmlTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteReplaceDmlTest.java @@ -619,7 +619,7 @@ public class CalciteReplaceDmlTest extends CalciteIngestionDmlTest // Use testQuery for EXPLAIN (not testIngestionQuery). testQuery( - new PlannerConfig(), + PlannerConfig.builder().useNativeQueryExplain(false).build(), ImmutableMap.of("sqlQueryId", "dummy"), Collections.emptyList(), StringUtils.format( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java index 7e9c96d6a00..a50e7212edd 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java @@ -45,6 +45,7 @@ import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.sql.SqlPlanningException; import org.apache.druid.sql.calcite.filtration.Filtration; +import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.util.CalciteTests; import org.joda.time.DateTime; @@ -545,7 +546,9 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest final String resources = "[]"; testQuery( + PlannerConfig.builder().useNativeQueryExplain(false).build(), query, + CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), ImmutableList.of( new Object[]{ @@ -1286,7 +1289,9 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; testQuery( + PlannerConfig.builder().useNativeQueryExplain(false).build(), query, + CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), ImmutableList.of( new Object[]{ diff --git a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java index 7ba0fb92a7c..20fe6578cbd 100644 --- a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java +++ b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java @@ -109,7 +109,6 @@ import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import javax.ws.rs.core.StreamingOutput; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -1185,7 +1184,12 @@ public class SqlResourceTest extends CalciteTestBase @Test public void testExplainCountStar() throws Exception { - Map queryContext = ImmutableMap.of(PlannerContext.CTX_SQL_QUERY_ID, DUMMY_SQL_QUERY_ID); + Map queryContext = ImmutableMap.of( + PlannerContext.CTX_SQL_QUERY_ID, + DUMMY_SQL_QUERY_ID, + PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN, + "false" + ); final List> rows = doPost( new SqlQuery( "EXPLAIN PLAN FOR SELECT COUNT(*) AS cnt FROM druid.foo", @@ -1203,8 +1207,10 @@ public class SqlResourceTest extends CalciteTestBase ImmutableMap.of( "PLAN", StringUtils.format( - "DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"context\":{\"sqlQueryId\":\"%s\"}}], signature=[{a0:LONG}])\n", - DUMMY_SQL_QUERY_ID + "DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"context\":{\"sqlQueryId\":\"%s\",\"%s\":\"%s\"}}], signature=[{a0:LONG}])\n", + DUMMY_SQL_QUERY_ID, + PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN, + "false" ), "RESOURCES", "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"