From 4fd4444b42df1c407400070b3e403d0f2523fabe Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Mon, 13 Nov 2017 20:24:06 -0800 Subject: [PATCH] SQL: Add "array" result format, and document result formats. (#5032) * SQL: Add "array" result format, and document result formats. * Code style. --- docs/content/querying/sql.md | 23 +++-- .../main/java/io/druid/sql/http/SqlQuery.java | 96 +++++++++++++++++-- .../java/io/druid/sql/http/SqlResource.java | 6 +- .../druid/sql/calcite/http/SqlQueryTest.java | 38 ++++++++ .../sql/calcite/http/SqlResourceTest.java | 65 +++++++++---- 5 files changed, 189 insertions(+), 39 deletions(-) create mode 100644 sql/src/test/java/io/druid/sql/calcite/http/SqlQueryTest.java diff --git a/docs/content/querying/sql.md b/docs/content/querying/sql.md index 8d6c5919b8b..e08ea043cbb 100644 --- a/docs/content/querying/sql.md +++ b/docs/content/querying/sql.md @@ -332,18 +332,25 @@ of configuration. ### JSON over HTTP You can make Druid SQL queries using JSON over HTTP by posting to the endpoint `/druid/v2/sql/`. The request should -be a JSON object with a "query" field, like `{"query" : "SELECT COUNT(*) FROM data_source WHERE foo = 'bar'"}`. You can -use _curl_ to send these queries from the command-line: +be a JSON object with a "query" field, like `{"query" : "SELECT COUNT(*) FROM data_source WHERE foo = 'bar'"}`. + +Results are available in two formats: "object" (the default; a JSON array of JSON objects), and "array" (a JSON array +of JSON arrays). In "object" form, each row's field names will match the column names from your SQL query. In "array" +form, each row's values are returned in the order specified in your SQL query. + +You can use _curl_ to send SQL queries from the command-line: ```bash $ cat query.json -{"query":"SELECT COUNT(*) FROM data_source"} +{"query":"SELECT COUNT(*) AS TheCount FROM data_source"} $ curl -XPOST -H'Content-Type: application/json' http://BROKER:8082/druid/v2/sql/ -d @query.json -[{"EXPR$0":24433}] +[{"TheCount":24433}] ``` -You can also provide [connection context parameters](#connection-context) by adding a "context" map, like: +Metadata is available over the HTTP API by querying the ["INFORMATION_SCHEMA" tables](#retrieving-metadata). + +Finally, you can also provide [connection context parameters](#connection-context) by adding a "context" map, like: ```json { @@ -354,8 +361,6 @@ You can also provide [connection context parameters](#connection-context) by add } ``` -Metadata is available over the HTTP API by querying the ["INFORMATION_SCHEMA" tables](#retrieving-metadata). - ### JDBC You can make Druid SQL queries using the [Avatica JDBC driver](https://calcite.apache.org/avatica/downloads/). Once @@ -404,6 +409,8 @@ Druid SQL supports setting connection parameters on the client. The parameters i All other context parameters you provide will be attached to Druid queries and can affect how they run. See [Query context](query-context.html) for details on the possible options. +Connection context can be specified as JDBC connection properties or as a "context" object in the JSON API. + |Parameter|Description|Default value| |---------|-----------|-------------| |`sqlTimeZone`|Sets the time zone for this connection, which will affect how time functions and timestamp literals behave. Should be a time zone name like "America/Los_Angeles" or offset like "-08:00".|UTC| @@ -411,8 +418,6 @@ All other context parameters you provide will be attached to Druid queries and c |`useApproximateTopN`|Whether to use approximate [TopN queries](topnquery.html) when a SQL query could be expressed as such. If false, exact [GroupBy queries](groupbyquery.html) will be used instead.|druid.sql.planner.useApproximateTopN on the broker| |`useFallback`|Whether to evaluate operations on the broker when they cannot be expressed as Druid queries. This option is not recommended for production since it can generate unscalable query plans. If false, SQL queries that cannot be translated to Druid queries will fail.|druid.sql.planner.useFallback on the broker| -Connection context can be specified as JDBC connection properties or as a "context" object in the JSON API. - ### Retrieving metadata Druid brokers infer table and column metadata for each dataSource from segments loaded in the cluster, and use this to diff --git a/sql/src/main/java/io/druid/sql/http/SqlQuery.java b/sql/src/main/java/io/druid/sql/http/SqlQuery.java index 5c1c0a87737..ce8c377dafd 100644 --- a/sql/src/main/java/io/druid/sql/http/SqlQuery.java +++ b/sql/src/main/java/io/druid/sql/http/SqlQuery.java @@ -21,23 +21,99 @@ package io.druid.sql.http; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonGenerator; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import io.druid.java.util.common.StringUtils; +import javax.annotation.Nullable; +import java.io.IOException; import java.util.Map; +import java.util.Objects; public class SqlQuery { + public enum ResultFormat + { + ARRAY { + @Override + public void writeResultStart(final JsonGenerator jsonGenerator) throws IOException + { + jsonGenerator.writeStartArray(); + } + + @Override + public void writeResultField( + final JsonGenerator jsonGenerator, + final String name, + final Object value + ) throws IOException + { + jsonGenerator.writeObject(value); + } + + @Override + public void writeResultEnd(final JsonGenerator jsonGenerator) throws IOException + { + jsonGenerator.writeEndArray(); + } + }, + + OBJECT { + @Override + public void writeResultStart(final JsonGenerator jsonGenerator) throws IOException + { + jsonGenerator.writeStartObject(); + } + + @Override + public void writeResultField( + final JsonGenerator jsonGenerator, + final String name, + final Object value + ) throws IOException + { + jsonGenerator.writeFieldName(name); + jsonGenerator.writeObject(value); + } + + @Override + public void writeResultEnd(final JsonGenerator jsonGenerator) throws IOException + { + jsonGenerator.writeEndObject(); + } + }; + + public abstract void writeResultStart(JsonGenerator jsonGenerator) throws IOException; + + public abstract void writeResultField(JsonGenerator jsonGenerator, String name, Object value) + throws IOException; + + public abstract void writeResultEnd(JsonGenerator jsonGenerator) throws IOException; + + @JsonCreator + public static ResultFormat fromString(@Nullable final String name) + { + if (name == null) { + return null; + } + return valueOf(StringUtils.toUpperCase(name)); + } + } + private final String query; + private final ResultFormat resultFormat; private final Map context; @JsonCreator public SqlQuery( @JsonProperty("query") final String query, + @JsonProperty("resultFormat") final ResultFormat resultFormat, @JsonProperty("context") final Map context ) { this.query = Preconditions.checkNotNull(query, "query"); + this.resultFormat = resultFormat == null ? ResultFormat.OBJECT : resultFormat; this.context = context == null ? ImmutableMap.of() : context; } @@ -47,6 +123,12 @@ public class SqlQuery return query; } + @JsonProperty + public ResultFormat getResultFormat() + { + return resultFormat; + } + @JsonProperty public Map getContext() { @@ -62,21 +144,16 @@ public class SqlQuery if (o == null || getClass() != o.getClass()) { return false; } - final SqlQuery sqlQuery = (SqlQuery) o; - - if (query != null ? !query.equals(sqlQuery.query) : sqlQuery.query != null) { - return false; - } - return context != null ? context.equals(sqlQuery.context) : sqlQuery.context == null; + return Objects.equals(query, sqlQuery.query) && + resultFormat == sqlQuery.resultFormat && + Objects.equals(context, sqlQuery.context); } @Override public int hashCode() { - int result = query != null ? query.hashCode() : 0; - result = 31 * result + (context != null ? context.hashCode() : 0); - return result; + return Objects.hash(query, resultFormat, context); } @Override @@ -84,6 +161,7 @@ public class SqlQuery { return "SqlQuery{" + "query='" + query + '\'' + + ", resultFormat=" + resultFormat + ", context=" + context + '}'; } diff --git a/sql/src/main/java/io/druid/sql/http/SqlResource.java b/sql/src/main/java/io/druid/sql/http/SqlResource.java index 8cc1d1f4d8d..dd73cd44c37 100644 --- a/sql/src/main/java/io/druid/sql/http/SqlResource.java +++ b/sql/src/main/java/io/druid/sql/http/SqlResource.java @@ -114,7 +114,7 @@ public class SqlResource while (!yielder.isDone()) { final Object[] row = yielder.get(); - jsonGenerator.writeStartObject(); + sqlQuery.getResultFormat().writeResultStart(jsonGenerator); for (int i = 0; i < fieldList.size(); i++) { final Object value; @@ -130,9 +130,9 @@ public class SqlResource value = row[i]; } - jsonGenerator.writeObjectField(fieldList.get(i).getName(), value); + sqlQuery.getResultFormat().writeResultField(jsonGenerator, fieldList.get(i).getName(), value); } - jsonGenerator.writeEndObject(); + sqlQuery.getResultFormat().writeResultEnd(jsonGenerator); yielder = yielder.next(null); } diff --git a/sql/src/test/java/io/druid/sql/calcite/http/SqlQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/http/SqlQueryTest.java new file mode 100644 index 00000000000..0445f54db10 --- /dev/null +++ b/sql/src/test/java/io/druid/sql/calcite/http/SqlQueryTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.sql.calcite.http; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import io.druid.segment.TestHelper; +import io.druid.sql.http.SqlQuery; +import org.junit.Assert; +import org.junit.Test; + +public class SqlQueryTest +{ + @Test + public void testSerde() throws Exception + { + final ObjectMapper jsonMapper = TestHelper.getJsonMapper(); + final SqlQuery query = new SqlQuery("SELECT 1", SqlQuery.ResultFormat.ARRAY, ImmutableMap.of("useCache", false)); + Assert.assertEquals(query, jsonMapper.readValue(jsonMapper.writeValueAsString(query), SqlQuery.class)); + } +} diff --git a/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java b/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java index 68d44524ae9..fd12039df69 100644 --- a/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java @@ -127,12 +127,28 @@ public class SqlResourceTest public void testCountStar() throws Exception { final List> rows = doPost( - new SqlQuery("SELECT COUNT(*) AS cnt FROM druid.foo", null) + new SqlQuery("SELECT COUNT(*) AS cnt, 'foo' AS TheFoo FROM druid.foo", null, null) ).rhs; Assert.assertEquals( ImmutableList.of( - ImmutableMap.of("cnt", 6) + ImmutableMap.of("cnt", 6, "TheFoo", "foo") + ), + rows + ); + } + + @Test + public void testCountStarAsArray() throws Exception + { + final List> rows = doPost( + new SqlQuery("SELECT COUNT(*), 'foo' FROM druid.foo", SqlQuery.ResultFormat.ARRAY, null), + new TypeReference>>() {} + ).rhs; + + Assert.assertEquals( + ImmutableList.of( + ImmutableList.of(6, "foo") ), rows ); @@ -142,7 +158,11 @@ public class SqlResourceTest public void testTimestampsInResponse() throws Exception { final List> rows = doPost( - new SqlQuery("SELECT __time, CAST(__time AS DATE) AS t2 FROM druid.foo LIMIT 1", null) + new SqlQuery( + "SELECT __time, CAST(__time AS DATE) AS t2 FROM druid.foo LIMIT 1", + SqlQuery.ResultFormat.OBJECT, + null + ) ).rhs; Assert.assertEquals( @@ -159,6 +179,7 @@ public class SqlResourceTest final List> rows = doPost( new SqlQuery( "SELECT __time, CAST(__time AS DATE) AS t2 FROM druid.foo LIMIT 1", + SqlQuery.ResultFormat.OBJECT, ImmutableMap.of(PlannerContext.CTX_SQL_TIME_ZONE, "America/Los_Angeles") ) ).rhs; @@ -175,7 +196,7 @@ public class SqlResourceTest public void testFieldAliasingSelect() throws Exception { final List> rows = doPost( - new SqlQuery("SELECT dim2 \"x\", dim2 \"y\" FROM druid.foo LIMIT 1", null) + new SqlQuery("SELECT dim2 \"x\", dim2 \"y\" FROM druid.foo LIMIT 1", SqlQuery.ResultFormat.OBJECT, null) ).rhs; Assert.assertEquals( @@ -190,7 +211,7 @@ public class SqlResourceTest public void testFieldAliasingGroupBy() throws Exception { final List> rows = doPost( - new SqlQuery("SELECT dim2 \"x\", dim2 \"y\" FROM druid.foo GROUP BY dim2", null) + new SqlQuery("SELECT dim2 \"x\", dim2 \"y\" FROM druid.foo GROUP BY dim2", SqlQuery.ResultFormat.OBJECT, null) ).rhs; Assert.assertEquals( @@ -207,7 +228,7 @@ public class SqlResourceTest public void testExplainCountStar() throws Exception { final List> rows = doPost( - new SqlQuery("EXPLAIN PLAN FOR SELECT COUNT(*) AS cnt FROM druid.foo", null) + new SqlQuery("EXPLAIN PLAN FOR SELECT COUNT(*) AS cnt FROM druid.foo", SqlQuery.ResultFormat.OBJECT, null) ).rhs; Assert.assertEquals( @@ -224,7 +245,13 @@ public class SqlResourceTest @Test public void testCannotValidate() throws Exception { - final QueryInterruptedException exception = doPost(new SqlQuery("SELECT dim3 FROM druid.foo", null)).lhs; + final QueryInterruptedException exception = doPost( + new SqlQuery( + "SELECT dim3 FROM druid.foo", + SqlQuery.ResultFormat.OBJECT, + null + ) + ).lhs; Assert.assertNotNull(exception); Assert.assertEquals(QueryInterruptedException.UNKNOWN_EXCEPTION, exception.getErrorCode()); @@ -237,7 +264,7 @@ public class SqlResourceTest { // SELECT + ORDER unsupported final QueryInterruptedException exception = doPost( - new SqlQuery("SELECT dim1 FROM druid.foo ORDER BY dim1", null) + new SqlQuery("SELECT dim1 FROM druid.foo ORDER BY dim1", SqlQuery.ResultFormat.OBJECT, null) ).lhs; Assert.assertNotNull(exception); @@ -255,9 +282,8 @@ public class SqlResourceTest final QueryInterruptedException exception = doPost( new SqlQuery( "SELECT DISTINCT dim1 FROM foo", - ImmutableMap.of( - "maxMergingDictionarySize", 1 - ) + SqlQuery.ResultFormat.OBJECT, + ImmutableMap.of("maxMergingDictionarySize", 1) ) ).lhs; @@ -267,7 +293,10 @@ public class SqlResourceTest } // Returns either an error or a result. - private Pair>> doPost(final SqlQuery query) throws Exception + private Pair doPost( + final SqlQuery query, + final TypeReference typeReference + ) throws Exception { final Response response = resource.doPost(query, req); if (response.getStatus() == 200) { @@ -276,12 +305,7 @@ public class SqlResourceTest output.write(baos); return Pair.of( null, - JSON_MAPPER.>>readValue( - baos.toByteArray(), - new TypeReference>>() - { - } - ) + JSON_MAPPER.readValue(baos.toByteArray(), typeReference) ); } else { return Pair.of( @@ -290,4 +314,9 @@ public class SqlResourceTest ); } } + + private Pair>> doPost(final SqlQuery query) throws Exception + { + return doPost(query, new TypeReference>>() {}); + } }