From f9ee2c353bccc9b76b6197f6a91a6ba0ff349c37 Mon Sep 17 00:00:00 2001 From: zachjsh Date: Fri, 9 Feb 2024 11:45:38 -0500 Subject: [PATCH] Extend the PARTITION BY clause to accept string literals for the time partitioning (#15836) This PR contains a portion of the changes from the inactive draft PR for integrating the catalog with the Calcite planner https://github.com/apache/druid/pull/13686 from @paul-rogers, extending the PARTITION BY clause to accept string literals for the time partitioning --- docs/multi-stage-query/reference.md | 27 ++++ sql/src/main/codegen/config.fmpp | 1 + sql/src/main/codegen/includes/common.ftl | 17 ++- sql/src/main/codegen/includes/insert.ftl | 6 +- sql/src/main/codegen/includes/replace.ftl | 7 +- .../sql/calcite/parser/DruidSqlIngest.java | 24 ++-- .../sql/calcite/parser/DruidSqlInsert.java | 49 +++++-- .../calcite/parser/DruidSqlParserUtils.java | 121 +++++++++++++++--- .../sql/calcite/parser/DruidSqlReplace.java | 62 +++++++-- .../calcite/parser/SqlGranularityLiteral.java | 77 +++++++++++ .../sql/calcite/planner/IngestHandler.java | 2 +- .../sql/calcite/CalciteInsertDmlTest.java | 45 ++++++- .../sql/calcite/CalciteReplaceDmlTest.java | 6 +- .../parser/DruidSqlParserUtilsTest.java | 84 +++++++----- ...rnalWithoutSecuritySupport-logicalPlan.txt | 2 +- .../ingest/httpExtern-logicalPlan.txt | 2 +- .../ingest/insertFromExternal-logicalPlan.txt | 2 +- ...tionedByP1DWithClusteredBy-logicalPlan.txt | 4 + .../insertWithClusteredBy-logicalPlan.txt | 2 +- .../insertWithPartitionedBy-logicalPlan.txt | 2 +- .../ingest/localExtern-logicalPlan.txt | 2 +- 21 files changed, 426 insertions(+), 118 deletions(-) create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/parser/SqlGranularityLiteral.java create mode 100644 sql/src/test/resources/calcite/expected/ingest/insertPartitionedByP1DWithClusteredBy-logicalPlan.txt diff --git a/docs/multi-stage-query/reference.md b/docs/multi-stage-query/reference.md index 25f55b31f74..f03ac25b157 100644 --- a/docs/multi-stage-query/reference.md +++ b/docs/multi-stage-query/reference.md @@ -287,6 +287,33 @@ The following ISO 8601 periods are supported for `TIME_FLOOR` and the string con - P3M - P1Y +The string constant can also include any of the keywords mentioned above: + +- `HOUR` - Same as `'PT1H'` +- `DAY` - Same as `'P1D'` +- `MONTH` - Same as `'P1M'` +- `YEAR` - Same as `'P1Y'` +- `ALL TIME` +- `ALL` - Alias for `ALL TIME` + +The `WEEK` granularity is deprecated and not supported in MSQ. + +Examples: + +```SQL +-- Keyword +PARTITIONED BY HOUR + +-- String literal +PARTITIONED BY 'HOUR' + +-- ISO 8601 period +PARTITIONED BY 'PT1H' + +-- TIME_FLOOR function +PARTITIONED BY TIME_FLOOR(__time, 'PT1H') +``` + For more information about partitioning, see [Partitioning](concepts.md#partitioning-by-time).

*Avoid partitioning by week, `P1W`, because weeks don't align neatly with months and years, making it difficult to partition by coarser granularities later. diff --git a/sql/src/main/codegen/config.fmpp b/sql/src/main/codegen/config.fmpp index 19ff322406e..12bd51351d2 100644 --- a/sql/src/main/codegen/config.fmpp +++ b/sql/src/main/codegen/config.fmpp @@ -54,6 +54,7 @@ data: { "org.apache.calcite.sql.SqlNodeList" "org.apache.calcite.sql.SqlBasicCall" "org.apache.druid.java.util.common.granularity.Granularity" + "org.apache.druid.java.util.common.granularity.GranularityType" "org.apache.druid.java.util.common.granularity.Granularities" "org.apache.druid.sql.calcite.parser.DruidSqlInsert" "org.apache.druid.sql.calcite.parser.DruidSqlParserUtils" diff --git a/sql/src/main/codegen/includes/common.ftl b/sql/src/main/codegen/includes/common.ftl index 757a869003e..95138a7dbbf 100644 --- a/sql/src/main/codegen/includes/common.ftl +++ b/sql/src/main/codegen/includes/common.ftl @@ -17,8 +17,7 @@ * under the License. */ -// Using fully qualified name for Pair class, since Calcite also has a same class name being used in the Parser.jj -org.apache.druid.java.util.common.Pair PartitionGranularity() : +SqlGranularityLiteral PartitionGranularity() : { SqlNode e; Granularity granularity; @@ -52,14 +51,14 @@ org.apache.druid.java.util.common.Pair PartitionGranularity | { - granularity = Granularities.ALL; - unparseString = "ALL"; + granularity = Granularities.ALL; + unparseString = "ALL"; } [ -