diff --git a/docs/querying/sql-window-functions.md b/docs/querying/sql-window-functions.md index d4293a852e7..197b269418e 100644 --- a/docs/querying/sql-window-functions.md +++ b/docs/querying/sql-window-functions.md @@ -28,10 +28,6 @@ description: Reference for window functions Apache Druid supports two query languages: [Druid SQL](sql.md) and [native queries](querying.md). This document describes the SQL language. -Window functions are an [experimental](../development/experimental.md) feature. -Development and testing are still at early stage. Feel free to try window functions and provide your feedback. -Windows functions are not currently supported by multi-stage-query engine so you cannot use them in SQL-based ingestion. - ::: Window functions in Apache Druid produce values based upon the relationship of one row within a window of rows to the other rows within the same window. A window is a group of related rows within a result set. For example, rows with the same value for a specific dimension. diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java index 2dbf3960c3a..cd2e9401954 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java @@ -90,6 +90,11 @@ public class PlannerContext */ public static final String CTX_SQL_OUTER_LIMIT = "sqlOuterLimit"; + /** + * Key to enable transfer of RACs over wire. + */ + public static final String CTX_ENABLE_RAC_TRANSFER_OVER_WIRE = "enableRACOverWire"; + /** * Context key for {@link PlannerContext#isUseBoundsAndSelectors()}. */ diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java index eb85dc83f30..f6757279294 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java @@ -1497,13 +1497,16 @@ public class DruidQuery // This would cause MSQ queries to plan as // Window over an inner scan and avoid // leaf operators + boolean pushLeafOperator = plannerContext.queryContext() + .getBoolean(PlannerContext.CTX_ENABLE_RAC_TRANSFER_OVER_WIRE, false) + && !plannerContext.featureAvailable(EngineFeature.WINDOW_LEAF_OPERATOR); return new WindowOperatorQuery( dataSource, new LegacySegmentSpec(Intervals.ETERNITY), plannerContext.queryContextMap(), windowing.getSignature(), operators, - plannerContext.featureAvailable(EngineFeature.WINDOW_LEAF_OPERATOR) ? ImmutableList.of() : null + pushLeafOperator ? null : ImmutableList.of() ); } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 412f378e8b5..7a038b83b9a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -16133,7 +16133,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest .run(); } - @NotYetSupported(Modes.UNSUPPORTED_DATASOURCE) @Test public void testWindowingOverJoin() { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteWindowQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteWindowQueryTest.java index fc86268ed59..1b9f2fe7ad4 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteWindowQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteWindowQueryTest.java @@ -37,6 +37,7 @@ import org.apache.druid.segment.column.RowSignature; import org.apache.druid.sql.calcite.CalciteWindowQueryTest.WindowQueryTestInputClass.TestType; import org.apache.druid.sql.calcite.QueryTestRunner.QueryResults; import org.apache.druid.sql.calcite.QueryVerification.QueryResultsVerifier; +import org.apache.druid.sql.calcite.planner.PlannerContext; import org.junit.Assert; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -296,10 +297,32 @@ public class CalciteWindowQueryTest extends BaseCalciteQueryTest ); assertEquals( - "Encountered a multi value column. Window processing does not support MVDs. " + "Encountered a multi value column [v0]. Window processing does not support MVDs. " + "Consider using UNNEST or MV_TO_ARRAY.", e.getMessage() ); + + final DruidException e1 = Assert.assertThrows( + DruidException.class, + () -> testBuilder() + .sql("select cityName, countryName, array_to_mv(array[1,length(cityName)]),\n" + + "row_number() over (partition by array_to_mv(array[1,length(cityName)]) order by countryName, cityName)\n" + + "from wikipedia\n" + + "where countryName in ('Austria', 'Republic of Korea') and cityName is not null\n" + + "order by 1, 2, 3") + .queryContext(ImmutableMap.of( + QueryContexts.ENABLE_DEBUG, true, + QueryContexts.CTX_SQL_STRINGIFY_ARRAYS, false, + PlannerContext.CTX_ENABLE_RAC_TRANSFER_OVER_WIRE, true + )) + .run() + ); + + assertEquals( + "Encountered a multi value column. Window processing does not support MVDs. " + + "Consider using UNNEST or MV_TO_ARRAY.", + e1.getMessage() + ); } private WindowOperatorQuery getWindowOperatorQuery(List> queries) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java b/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java index 1a83a708e21..2939d30e735 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java @@ -89,7 +89,6 @@ public @interface NotYetSupported RESULT_MISMATCH(AssertionError.class, "(assertResulEquals|AssertionError: column content mismatch)"), LONG_CASTING(AssertionError.class, "expected: java.lang.Long"), UNSUPPORTED_NULL_ORDERING(DruidException.class, "(A|DE)SCENDING ordering with NULLS (LAST|FIRST)"), - UNSUPPORTED_DATASOURCE(DruidException.class, "WindowOperatorQuery must run on top of a query or inline data source"), UNION_WITH_COMPLEX_OPERAND(DruidException.class, "Only Table and Values are supported as inputs for Union"), UNION_MORE_STRICT_ROWTYPE_CHECK(DruidException.class, "Row signature mismatch in Union inputs"), UNNEST_NOT_SUPPORTED_CORRELATE_CONVERSION(DruidException.class, "Missing conversion( is|s are) LogicalCorrelate"),