Apache/druid
1
0
Fork 0
mirror of https://github.com/apache/druid.git synced 2025-02-17 07:25:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

SQL: Add "array" result format, and document result formats. (#5032)

Browse Source 
* SQL: Add "array" result format, and document result formats.

* Code style.
This commit is contained in:
Gian Merlino 2017-11-13 20:24:06 -08:00 committed by Fangjin Yang
parent 6c0c858913
commit 4fd4444b42
5 changed files with 189 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
docs/content/querying/sql.md
View File

@ -332,18 +332,25 @@ of configuration.
### JSON over HTTP
You can make Druid SQL queries using JSON over HTTP by posting to the endpoint `/druid/v2/sql/`. The request should
be a JSON object with a "query" field, like `{"query" : "SELECT COUNT(*) FROM data_source WHERE foo = 'bar'"}`. You can
use _curl_ to send these queries from the command-line:
be a JSON object with a "query" field, like `{"query" : "SELECT COUNT(*) FROM data_source WHERE foo = 'bar'"}`.
Results are available in two formats: "object" (the default; a JSON array of JSON objects), and "array" (a JSON array
of JSON arrays). In "object" form, each row's field names will match the column names from your SQL query. In "array"
form, each row's values are returned in the order specified in your SQL query.
You can use _curl_ to send SQL queries from the command-line:
```bash
$ cat query.json
{"query":"SELECT COUNT(*) FROM data_source"}
{"query":"SELECT COUNT(*) AS TheCount FROM data_source"}
$ curl -XPOST -H'Content-Type: application/json' http://BROKER:8082/druid/v2/sql/ -d @query.json
[{"EXPR$0":24433}]
[{"TheCount":24433}]
```
You can also provide [connection context parameters](#connection-context) by adding a "context" map, like:
Metadata is available over the HTTP API by querying the ["INFORMATION_SCHEMA" tables](#retrieving-metadata).
Finally, you can also provide [connection context parameters](#connection-context) by adding a "context" map, like:
```json
{
@ -354,8 +361,6 @@ You can also provide [connection context parameters](#connection-context) by add
}
```
Metadata is available over the HTTP API by querying the ["INFORMATION_SCHEMA" tables](#retrieving-metadata).
### JDBC
You can make Druid SQL queries using the [Avatica JDBC driver](https://calcite.apache.org/avatica/downloads/). Once
@ -404,6 +409,8 @@ Druid SQL supports setting connection parameters on the client. The parameters i
All other context parameters you provide will be attached to Druid queries and can affect how they run. See
[Query context](query-context.html) for details on the possible options.
Connection context can be specified as JDBC connection properties or as a "context" object in the JSON API.
|Parameter|Description|Default value|
|---------|-----------|-------------|
|`sqlTimeZone`|Sets the time zone for this connection, which will affect how time functions and timestamp literals behave. Should be a time zone name like "America/Los_Angeles" or offset like "-08:00".|UTC|
@ -411,8 +418,6 @@ All other context parameters you provide will be attached to Druid queries and c
|`useApproximateTopN`|Whether to use approximate [TopN queries](topnquery.html) when a SQL query could be expressed as such. If false, exact [GroupBy queries](groupbyquery.html) will be used instead.|druid.sql.planner.useApproximateTopN on the broker|
|`useFallback`|Whether to evaluate operations on the broker when they cannot be expressed as Druid queries. This option is not recommended for production since it can generate unscalable query plans. If false, SQL queries that cannot be translated to Druid queries will fail.|druid.sql.planner.useFallback on the broker|
Connection context can be specified as JDBC connection properties or as a "context" object in the JSON API.
### Retrieving metadata
Druid brokers infer table and column metadata for each dataSource from segments loaded in the cluster, and use this to

96
sql/src/main/java/io/druid/sql/http/SqlQuery.java
View File

@ -21,23 +21,99 @@ package io.druid.sql.http;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonGenerator;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import io.druid.java.util.common.StringUtils;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.Map;
import java.util.Objects;
public class SqlQuery
{
public enum ResultFormat
{
ARRAY {
@Override
public void writeResultStart(final JsonGenerator jsonGenerator) throws IOException
{
jsonGenerator.writeStartArray();
}
@Override
public void writeResultField(
final JsonGenerator jsonGenerator,
final String name,
final Object value
) throws IOException
{
jsonGenerator.writeObject(value);
}
@Override
public void writeResultEnd(final JsonGenerator jsonGenerator) throws IOException
{
jsonGenerator.writeEndArray();
}
},
OBJECT {
@Override
public void writeResultStart(final JsonGenerator jsonGenerator) throws IOException
{
jsonGenerator.writeStartObject();
}
@Override
public void writeResultField(
final JsonGenerator jsonGenerator,
final String name,
final Object value
) throws IOException
{
jsonGenerator.writeFieldName(name);
jsonGenerator.writeObject(value);
}
@Override
public void writeResultEnd(final JsonGenerator jsonGenerator) throws IOException
{
jsonGenerator.writeEndObject();
}
};
public abstract void writeResultStart(JsonGenerator jsonGenerator) throws IOException;
public abstract void writeResultField(JsonGenerator jsonGenerator, String name, Object value)
throws IOException;
public abstract void writeResultEnd(JsonGenerator jsonGenerator) throws IOException;
@JsonCreator
public static ResultFormat fromString(@Nullable final String name)
{
if (name == null) {
return null;
}
return valueOf(StringUtils.toUpperCase(name));
}
}
private final String query;
private final ResultFormat resultFormat;
private final Map<String, Object> context;
@JsonCreator
public SqlQuery(
@JsonProperty("query") final String query,
@JsonProperty("resultFormat") final ResultFormat resultFormat,
@JsonProperty("context") final Map<String, Object> context
)
{
this.query = Preconditions.checkNotNull(query, "query");
this.resultFormat = resultFormat == null ? ResultFormat.OBJECT : resultFormat;
this.context = context == null ? ImmutableMap.<String, Object>of() : context;
}
@ -47,6 +123,12 @@ public class SqlQuery
return query;
}
@JsonProperty
public ResultFormat getResultFormat()
{
return resultFormat;
}
@JsonProperty
public Map<String, Object> getContext()
{
@ -62,21 +144,16 @@ public class SqlQuery
if (o == null || getClass() != o.getClass()) {
return false;
}
final SqlQuery sqlQuery = (SqlQuery) o;
if (query != null ? !query.equals(sqlQuery.query) : sqlQuery.query != null) {
return false;
}
return context != null ? context.equals(sqlQuery.context) : sqlQuery.context == null;
return Objects.equals(query, sqlQuery.query) &&
resultFormat == sqlQuery.resultFormat &&
Objects.equals(context, sqlQuery.context);
}
@Override
public int hashCode()
{
int result = query != null ? query.hashCode() : 0;
result = 31 * result + (context != null ? context.hashCode() : 0);
return result;
return Objects.hash(query, resultFormat, context);
}
@Override
@ -84,6 +161,7 @@ public class SqlQuery
{
return "SqlQuery{" +
"query='" + query + '\'' +
", resultFormat=" + resultFormat +
", context=" + context +
'}';
}

6
sql/src/main/java/io/druid/sql/http/SqlResource.java
View File

@ -114,7 +114,7 @@ public class SqlResource
while (!yielder.isDone()) {
final Object[] row = yielder.get();
jsonGenerator.writeStartObject();
sqlQuery.getResultFormat().writeResultStart(jsonGenerator);
for (int i = 0; i < fieldList.size(); i++) {
final Object value;
@ -130,9 +130,9 @@ public class SqlResource
value = row[i];
}
jsonGenerator.writeObjectField(fieldList.get(i).getName(), value);
sqlQuery.getResultFormat().writeResultField(jsonGenerator, fieldList.get(i).getName(), value);
}
jsonGenerator.writeEndObject();
sqlQuery.getResultFormat().writeResultEnd(jsonGenerator);
yielder = yielder.next(null);
}

38
sql/src/test/java/io/druid/sql/calcite/http/SqlQueryTest.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.sql.calcite.http;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableMap;
import io.druid.segment.TestHelper;
import io.druid.sql.http.SqlQuery;
import org.junit.Assert;
import org.junit.Test;
public class SqlQueryTest
{
@Test
public void testSerde() throws Exception
{
final ObjectMapper jsonMapper = TestHelper.getJsonMapper();
final SqlQuery query = new SqlQuery("SELECT 1", SqlQuery.ResultFormat.ARRAY, ImmutableMap.of("useCache", false));
Assert.assertEquals(query, jsonMapper.readValue(jsonMapper.writeValueAsString(query), SqlQuery.class));
}
}

65
sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java
View File

@ -127,12 +127,28 @@ public class SqlResourceTest
public void testCountStar() throws Exception
{
final List<Map<String, Object>> rows = doPost(
new SqlQuery("SELECT COUNT(*) AS cnt FROM druid.foo", null)
new SqlQuery("SELECT COUNT(*) AS cnt, 'foo' AS TheFoo FROM druid.foo", null, null)
).rhs;
Assert.assertEquals(
ImmutableList.of(
ImmutableMap.of("cnt", 6)
ImmutableMap.of("cnt", 6, "TheFoo", "foo")
),
rows
);
}
@Test
public void testCountStarAsArray() throws Exception
{
final List<List<Object>> rows = doPost(
new SqlQuery("SELECT COUNT(*), 'foo' FROM druid.foo", SqlQuery.ResultFormat.ARRAY, null),
new TypeReference<List<List<Object>>>() {}
).rhs;
Assert.assertEquals(
ImmutableList.of(
ImmutableList.of(6, "foo")
),
rows
);
@ -142,7 +158,11 @@ public class SqlResourceTest
public void testTimestampsInResponse() throws Exception
{
final List<Map<String, Object>> rows = doPost(
new SqlQuery("SELECT __time, CAST(__time AS DATE) AS t2 FROM druid.foo LIMIT 1", null)
new SqlQuery(
"SELECT __time, CAST(__time AS DATE) AS t2 FROM druid.foo LIMIT 1",
SqlQuery.ResultFormat.OBJECT,
null
)
).rhs;
Assert.assertEquals(
@ -159,6 +179,7 @@ public class SqlResourceTest
final List<Map<String, Object>> rows = doPost(
new SqlQuery(
"SELECT __time, CAST(__time AS DATE) AS t2 FROM druid.foo LIMIT 1",
SqlQuery.ResultFormat.OBJECT,
ImmutableMap.<String, Object>of(PlannerContext.CTX_SQL_TIME_ZONE, "America/Los_Angeles")
)
).rhs;
@ -175,7 +196,7 @@ public class SqlResourceTest
public void testFieldAliasingSelect() throws Exception
{
final List<Map<String, Object>> rows = doPost(
new SqlQuery("SELECT dim2 \"x\", dim2 \"y\" FROM druid.foo LIMIT 1", null)
new SqlQuery("SELECT dim2 \"x\", dim2 \"y\" FROM druid.foo LIMIT 1", SqlQuery.ResultFormat.OBJECT, null)
).rhs;
Assert.assertEquals(
@ -190,7 +211,7 @@ public class SqlResourceTest
public void testFieldAliasingGroupBy() throws Exception
{
final List<Map<String, Object>> rows = doPost(
new SqlQuery("SELECT dim2 \"x\", dim2 \"y\" FROM druid.foo GROUP BY dim2", null)
new SqlQuery("SELECT dim2 \"x\", dim2 \"y\" FROM druid.foo GROUP BY dim2", SqlQuery.ResultFormat.OBJECT, null)
).rhs;
Assert.assertEquals(
@ -207,7 +228,7 @@ public class SqlResourceTest
public void testExplainCountStar() throws Exception
{
final List<Map<String, Object>> rows = doPost(
new SqlQuery("EXPLAIN PLAN FOR SELECT COUNT(*) AS cnt FROM druid.foo", null)
new SqlQuery("EXPLAIN PLAN FOR SELECT COUNT(*) AS cnt FROM druid.foo", SqlQuery.ResultFormat.OBJECT, null)
).rhs;
Assert.assertEquals(
@ -224,7 +245,13 @@ public class SqlResourceTest
@Test
public void testCannotValidate() throws Exception
{
final QueryInterruptedException exception = doPost(new SqlQuery("SELECT dim3 FROM druid.foo", null)).lhs;
final QueryInterruptedException exception = doPost(
new SqlQuery(
"SELECT dim3 FROM druid.foo",
SqlQuery.ResultFormat.OBJECT,
null
)
).lhs;
Assert.assertNotNull(exception);
Assert.assertEquals(QueryInterruptedException.UNKNOWN_EXCEPTION, exception.getErrorCode());
@ -237,7 +264,7 @@ public class SqlResourceTest
{
// SELECT + ORDER unsupported
final QueryInterruptedException exception = doPost(
new SqlQuery("SELECT dim1 FROM druid.foo ORDER BY dim1", null)
new SqlQuery("SELECT dim1 FROM druid.foo ORDER BY dim1", SqlQuery.ResultFormat.OBJECT, null)
).lhs;
Assert.assertNotNull(exception);
@ -255,9 +282,8 @@ public class SqlResourceTest
final QueryInterruptedException exception = doPost(
new SqlQuery(
"SELECT DISTINCT dim1 FROM foo",
ImmutableMap.<String, Object>of(
"maxMergingDictionarySize", 1
)
SqlQuery.ResultFormat.OBJECT,
ImmutableMap.<String, Object>of("maxMergingDictionarySize", 1)
)
).lhs;
@ -267,7 +293,10 @@ public class SqlResourceTest
}
// Returns either an error or a result.
private Pair<QueryInterruptedException, List<Map<String, Object>>> doPost(final SqlQuery query) throws Exception
private <T> Pair<QueryInterruptedException, T> doPost(
final SqlQuery query,
final TypeReference<T> typeReference
) throws Exception
{
final Response response = resource.doPost(query, req);
if (response.getStatus() == 200) {
@ -276,12 +305,7 @@ public class SqlResourceTest
output.write(baos);
return Pair.of(
null,
JSON_MAPPER.<List<Map<String, Object>>>readValue(
baos.toByteArray(),
new TypeReference<List<Map<String, Object>>>()
{
}
)
JSON_MAPPER.<T>readValue(baos.toByteArray(), typeReference)
);
} else {
return Pair.of(
@ -290,4 +314,9 @@ public class SqlResourceTest
);
}
}
private Pair<QueryInterruptedException, List<Map<String, Object>>> doPost(final SqlQuery query) throws Exception
{
return doPost(query, new TypeReference<List<Map<String, Object>>>() {});
}
}