Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make it easier for devs to add CalciteQueryTests (#9922) 

* Add ingestion specs for CalciteQueryTests

This PR introduces ingestion specs that can be used for local testing
so that CalciteQueryTests can be built on a druid cluster.

* Add README

* Update sql/src/test/resources/calcite/tests/README.md
This commit is contained in:
Suneet Saldanha 2020-05-27 13:03:53 -07:00 committed by GitHub
parent b8c2266aa0
commit b5dfa5322b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 410 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

342
sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java
View File

@ -124,6 +124,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executor;
import java.util.function.BooleanSupplier;
import java.util.stream.Collectors;
/**
* Utility functions for Calcite tests.
@ -309,189 +310,178 @@ public class CalciteTests
.withRollup(false)
.build();
public static final List<InputRow> ROWS1 = ImmutableList.of(
createRow(
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-01")
.put("m1", "1.0")
.put("m2", "1.0")
.put("dim1", "")
.put("dim2", ImmutableList.of("a"))
.put("dim3", ImmutableList.of("a", "b"))
.build()
),
createRow(
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-02")
.put("m1", "2.0")
.put("m2", "2.0")
.put("dim1", "10.1")
.put("dim2", ImmutableList.of())
.put("dim3", ImmutableList.of("b", "c"))
.build()
),
createRow(
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-03")
.put("m1", "3.0")
.put("m2", "3.0")
.put("dim1", "2")
.put("dim2", ImmutableList.of(""))
.put("dim3", ImmutableList.of("d"))
.build()
),
createRow(
ImmutableMap.<String, Object>builder()
.put("t", "2001-01-01")
.put("m1", "4.0")
.put("m2", "4.0")
.put("dim1", "1")
.put("dim2", ImmutableList.of("a"))
.put("dim3", ImmutableList.of(""))
.build()
),
createRow(
ImmutableMap.<String, Object>builder()
.put("t", "2001-01-02")
.put("m1", "5.0")
.put("m2", "5.0")
.put("dim1", "def")
.put("dim2", ImmutableList.of("abc"))
.put("dim3", ImmutableList.of())
.build()
),
createRow(
ImmutableMap.<String, Object>builder()
.put("t", "2001-01-03")
.put("m1", "6.0")
.put("m2", "6.0")
.put("dim1", "abc")
.build()
)
public static final List<ImmutableMap<String, Object>> RAW_ROWS1 = ImmutableList.of(
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-01")
.put("m1", "1.0")
.put("m2", "1.0")
.put("dim1", "")
.put("dim2", ImmutableList.of("a"))
.put("dim3", ImmutableList.of("a", "b"))
.build(),
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-02")
.put("m1", "2.0")
.put("m2", "2.0")
.put("dim1", "10.1")
.put("dim2", ImmutableList.of())
.put("dim3", ImmutableList.of("b", "c"))
.build(),
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-03")
.put("m1", "3.0")
.put("m2", "3.0")
.put("dim1", "2")
.put("dim2", ImmutableList.of(""))
.put("dim3", ImmutableList.of("d"))
.build(),
ImmutableMap.<String, Object>builder()
.put("t", "2001-01-01")
.put("m1", "4.0")
.put("m2", "4.0")
.put("dim1", "1")
.put("dim2", ImmutableList.of("a"))
.put("dim3", ImmutableList.of(""))
.build(),
ImmutableMap.<String, Object>builder()
.put("t", "2001-01-02")
.put("m1", "5.0")
.put("m2", "5.0")
.put("dim1", "def")
.put("dim2", ImmutableList.of("abc"))
.put("dim3", ImmutableList.of())
.build(),
ImmutableMap.<String, Object>builder()
.put("t", "2001-01-03")
.put("m1", "6.0")
.put("m2", "6.0")
.put("dim1", "abc")
.build()
);
public static final List<InputRow> ROWS1 =
RAW_ROWS1.stream().map(CalciteTests::createRow).collect(Collectors.toList());
public static final List<InputRow> ROWS1_WITH_NUMERIC_DIMS = ImmutableList.of(
createRow(
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-01")
.put("m1", "1.0")
.put("m2", "1.0")
.put("d1", 1.0)
.put("f1", 1.0f)
.put("l1", 7L)
.put("dim1", "")
.put("dim2", ImmutableList.of("a"))
.put("dim3", ImmutableList.of("a", "b"))
.put("dim4", "a")
.put("dim5", "aa")
.build(),
PARSER_NUMERIC_DIMS
),
createRow(
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-02")
.put("m1", "2.0")
.put("m2", "2.0")
.put("d1", 1.7)
.put("d2", 1.7)
.put("f1", 0.1f)
.put("f2", 0.1f)
.put("l1", 325323L)
.put("l2", 325323L)
.put("dim1", "10.1")
.put("dim2", ImmutableList.of())
.put("dim3", ImmutableList.of("b", "c"))
.put("dim4", "a")
.put("dim5", "ab")
.build(),
PARSER_NUMERIC_DIMS
),
createRow(
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-03")
.put("m1", "3.0")
.put("m2", "3.0")
.put("d1", 0.0)
.put("d2", 0.0)
.put("f1", 0.0)
.put("f2", 0.0)
.put("l1", 0)
.put("l2", 0)
.put("dim1", "2")
.put("dim2", ImmutableList.of(""))
.put("dim3", ImmutableList.of("d"))
.put("dim4", "a")
.put("dim5", "ba")
.build(),
PARSER_NUMERIC_DIMS
),
createRow(
ImmutableMap.<String, Object>builder()
.put("t", "2001-01-01")
.put("m1", "4.0")
.put("m2", "4.0")
.put("dim1", "1")
.put("dim2", ImmutableList.of("a"))
.put("dim3", ImmutableList.of(""))
.put("dim4", "b")
.put("dim5", "ad")
.build(),
PARSER_NUMERIC_DIMS
),
createRow(
ImmutableMap.<String, Object>builder()
.put("t", "2001-01-02")
.put("m1", "5.0")
.put("m2", "5.0")
.put("dim1", "def")
.put("dim2", ImmutableList.of("abc"))
.put("dim3", ImmutableList.of())
.put("dim4", "b")
.put("dim5", "aa")
.build(),
PARSER_NUMERIC_DIMS
),
createRow(
ImmutableMap.<String, Object>builder()
.put("t", "2001-01-03")
.put("m1", "6.0")
.put("m2", "6.0")
.put("dim1", "abc")
.put("dim4", "b")
.put("dim5", "ab")
.build(),
PARSER_NUMERIC_DIMS
)
public static final List<ImmutableMap<String, Object>> RAW_ROWS1_WITH_NUMERIC_DIMS = ImmutableList.of(
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-01")
.put("m1", "1.0")
.put("m2", "1.0")
.put("d1", 1.0)
.put("f1", 1.0f)
.put("l1", 7L)
.put("dim1", "")
.put("dim2", ImmutableList.of("a"))
.put("dim3", ImmutableList.of("a", "b"))
.put("dim4", "a")
.put("dim5", "aa")
.build(),
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-02")
.put("m1", "2.0")
.put("m2", "2.0")
.put("d1", 1.7)
.put("d2", 1.7)
.put("f1", 0.1f)
.put("f2", 0.1f)
.put("l1", 325323L)
.put("l2", 325323L)
.put("dim1", "10.1")
.put("dim2", ImmutableList.of())
.put("dim3", ImmutableList.of("b", "c"))
.put("dim4", "a")
.put("dim5", "ab")
.build(),
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-03")
.put("m1", "3.0")
.put("m2", "3.0")
.put("d1", 0.0)
.put("d2", 0.0)
.put("f1", 0.0)
.put("f2", 0.0)
.put("l1", 0)
.put("l2", 0)
.put("dim1", "2")
.put("dim2", ImmutableList.of(""))
.put("dim3", ImmutableList.of("d"))
.put("dim4", "a")
.put("dim5", "ba")
.build(),
ImmutableMap.<String, Object>builder()
.put("t", "2001-01-01")
.put("m1", "4.0")
.put("m2", "4.0")
.put("dim1", "1")
.put("dim2", ImmutableList.of("a"))
.put("dim3", ImmutableList.of(""))
.put("dim4", "b")
.put("dim5", "ad")
.build(),
ImmutableMap.<String, Object>builder()
.put("t", "2001-01-02")
.put("m1", "5.0")
.put("m2", "5.0")
.put("dim1", "def")
.put("dim2", ImmutableList.of("abc"))
.put("dim3", ImmutableList.of())
.put("dim4", "b")
.put("dim5", "aa")
.build(),
ImmutableMap.<String, Object>builder()
.put("t", "2001-01-03")
.put("m1", "6.0")
.put("m2", "6.0")
.put("dim1", "abc")
.put("dim4", "b")
.put("dim5", "ab")
.build()
);
public static final List<InputRow> ROWS1_WITH_NUMERIC_DIMS =
RAW_ROWS1_WITH_NUMERIC_DIMS.stream().map(raw -> createRow(raw, PARSER_NUMERIC_DIMS)).collect(Collectors.toList());
public static final List<InputRow> ROWS2 = ImmutableList.of(
createRow("2000-01-01", "דרואיד", "he", 1.0),
createRow("2000-01-01", "druid", "en", 1.0),
createRow("2000-01-01", "друид", "ru", 1.0)
public static final List<ImmutableMap<String, Object>> RAW_ROWS2 = ImmutableList.of(
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-01")
.put("dim1", "דרואיד")
.put("dim2", "he")
.put("m1", 1.0)
.build(),
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-01")
.put("dim1", "druid")
.put("dim2", "en")
.put("m1", 1.0)
.build(),
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-01")
.put("dim1", "друид")
.put("dim2", "ru")
.put("m1", 1.0)
.build()
);
public static final List<InputRow> ROWS2 =
RAW_ROWS2.stream().map(CalciteTests::createRow).collect(Collectors.toList());
public static final List<InputRow> ROWS1_WITH_FULL_TIMESTAMP = ImmutableList.of(
createRow(
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-01T10:51:45.695Z")
.put("m1", "1.0")
.put("m2", "1.0")
.put("dim1", "")
.put("dim2", ImmutableList.of("a"))
.put("dim3", ImmutableList.of("a", "b"))
.build()
),
createRow(
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-18T10:51:45.695Z")
.put("m1", "2.0")
.put("m2", "2.0")
.put("dim1", "10.1")
.put("dim2", ImmutableList.of())
.put("dim3", ImmutableList.of("b", "c"))
.build()
)
public static final List<ImmutableMap<String, Object>> RAW_ROWS1_WITH_FULL_TIMESTAMP = ImmutableList.of(
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-01T10:51:45.695Z")
.put("m1", "1.0")
.put("m2", "1.0")
.put("dim1", "")
.put("dim2", ImmutableList.of("a"))
.put("dim3", ImmutableList.of("a", "b"))
.build(),
ImmutableMap.<String, Object>builder()
.put("t", "2000-01-18T10:51:45.695Z")
.put("m1", "2.0")
.put("m2", "2.0")
.put("dim1", "10.1")
.put("dim2", ImmutableList.of())
.put("dim3", ImmutableList.of("b", "c"))
.build()
);
public static final List<InputRow> ROWS1_WITH_FULL_TIMESTAMP =
RAW_ROWS1_WITH_FULL_TIMESTAMP.stream().map(CalciteTests::createRow).collect(Collectors.toList());
public static final List<InputRow> FORBIDDEN_ROWS = ImmutableList.of(

7
sql/src/test/resources/calcite/tests/README.md Normal file
View File

@ -0,0 +1,7 @@
This package contains ingestion specs for datasources used by Calcite Tests.
The purpose of these files is to make it easier to look at and manipulate the data under test so that you can easily
validate if the results of the SQL query written are as expected.
> NOTE: The provided specs are not guaranteed to be in sync with the datasources used by the test. The source of truth
> is org.apache.druid.sql.calcite.util.CalciteTests

53
sql/src/test/resources/calcite/tests/foo.json Normal file
View File

@ -0,0 +1,53 @@
{
"type": "index_parallel",
"spec": {
"ioConfig": {
"type": "index_parallel",
"inputSource": {
"type": "inline",
"data": "{\"t\":\"2000-01-01\",\"m1\":\"1.0\",\"m2\":\"1.0\",\"dim1\":\"\",\"dim2\":[\"a\"],\"dim3\":[\"a\",\"b\"]},\n{\"t\":\"2000-01-02\",\"m1\":\"2.0\",\"m2\":\"2.0\",\"dim1\":\"10.1\",\"dim2\":[],\"dim3\":[\"b\",\"c\"]},\n{\"t\":\"2000-01-03\",\"m1\":\"3.0\",\"m2\":\"3.0\",\"dim1\":\"2\",\"dim2\":[\"\"],\"dim3\":[\"d\"]},\n{\"t\":\"2001-01-01\",\"m1\":\"4.0\",\"m2\":\"4.0\",\"dim1\":\"1\",\"dim2\":[\"a\"],\"dim3\":[\"\"]},\n{\"t\":\"2001-01-02\",\"m1\":\"5.0\",\"m2\":\"5.0\",\"dim1\":\"def\",\"dim2\":[\"abc\"],\"dim3\":[]},\n{\"t\":\"2001-01-03\",\"m1\":\"6.0\",\"m2\":\"6.0\",\"dim1\":\"abc\"}"
},
"inputFormat": {
"type": "json"
}
},
"tuningConfig": {
"type": "index_parallel",
"partitionsSpec": {
"type": "dynamic"
}
},
"dataSchema": {
"dataSource": "foo",
"granularitySpec": {
"type": "uniform",
"queryGranularity": "NONE",
"rollup": false,
"segmentGranularity": "YEAR"
},
"timestampSpec": {
"column": "t",
"format": "auto"
},
"dimensionsSpec": {
"dimensions": [
"dim1",
"dim2",
"dim3"
]
},
"metricsSpec": [
{
"name": "m1",
"type": "floatSum",
"fieldName": "m1"
},
{
"name": "m2",
"type": "doubleSum",
"fieldName": "m2"
}
]
}
}
}

47
sql/src/test/resources/calcite/tests/foo2.json Normal file
View File

@ -0,0 +1,47 @@
{
"type": "index_parallel",
"spec": {
"ioConfig": {
"type": "index_parallel",
"inputSource": {
"type": "inline",
"data": "{\"t\":\"2000-01-01\",\"dim1\":\"דרואיד\",\"dim2\":\"he\",\"m1\":1.0}\n{\"t\":\"2000-01-01\",\"dim1\":\"druid\",\"dim2\":\"en\",\"m1\":1.0}\n{\"t\":\"2000-01-01\",\"dim1\":\"друид\",\"dim2\":\"ru\",\"m1\":1.0}"
},
"inputFormat": {
"type": "json"
}
},
"tuningConfig": {
"type": "index_parallel",
"partitionsSpec": {
"type": "dynamic"
}
},
"dataSchema": {
"dataSource": "foo2",
"granularitySpec": {
"type": "uniform",
"queryGranularity": "NONE",
"rollup": false,
"segmentGranularity": "YEAR"
},
"timestampSpec": {
"column": "t",
"format": "auto"
},
"dimensionsSpec": {
"dimensions": [
"dim1",
"dim2"
]
},
"metricsSpec": [
{
"name": "m1",
"type": "longSum",
"fieldName": "m1"
}
]
}
}
}

52
sql/src/test/resources/calcite/tests/foo4.json Normal file
View File

@ -0,0 +1,52 @@
{
"type": "index_parallel",
"spec": {
"ioConfig": {
"type": "index_parallel",
"inputSource": {
"type": "inline",
"data": "{\"t\":\"2000-01-01T10:51:45.695Z\",\"m1\":\"1.0\",\"m2\":\"1.0\",\"dim1\":\"\",\"dim2\":[\"a\"],\"dim3\":[\"a\",\"b\"]}\n{\"t\":\"2000-01-18T10:51:45.695Z\",\"m1\":\"2.0\",\"m2\":\"2.0\",\"dim1\":\"10.1\",\"dim2\":[],\"dim3\":[\"b\",\"c\"]}"
},
"inputFormat": {
"type": "json"
}
},
"tuningConfig": {
"type": "index_parallel",
"partitionsSpec": {
"type": "dynamic"
}
},
"dataSchema": {
"dataSource": "foo4",
"granularitySpec": {
"type": "uniform",
"queryGranularity": "HOUR",
"rollup": true,
"segmentGranularity": "YEAR"
},
"timestampSpec": {
"column": "t",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
"dim2",
"dim3"
]
},
"metricsSpec": [
{
"name": "m1",
"type": "floatSum",
"fieldName": "m1"
},
{
"name": "m2",
"type": "doubleSum",
"fieldName": "m2"
}
]
}
}
}

6
sql/src/test/resources/calcite/tests/lookyloo.json Normal file
View File

@ -0,0 +1,6 @@
{
"a": "xa",
"abc": "xabc",
"nosuchkey": "mysteryvalue",
"6": "x6"
}

79
sql/src/test/resources/calcite/tests/numFoo.json Normal file
View File

@ -0,0 +1,79 @@
{
"type": "index_parallel",
"spec": {
"ioConfig": {
"type": "index_parallel",
"inputSource": {
"type": "inline",
"data": "{\"t\":\"2000-01-01\",\"m1\":\"1.0\",\"m2\":\"1.0\",\"d1\":1.0,\"f1\":1.0,\"l1\":7,\"dim1\":\"\",\"dim2\":[\"a\"],\"dim3\":[\"a\",\"b\"],\"dim4\":\"a\",\"dim5\":\"aa\"}\n{\"t\":\"2000-01-02\",\"m1\":\"2.0\",\"m2\":\"2.0\",\"d1\":1.7,\"d2\":1.7,\"f1\":0.1,\"f2\":0.1,\"l1\":325323,\"l2\":325323,\"dim1\":\"10.1\",\"dim2\":[],\"dim3\":[\"b\",\"c\"],\"dim4\":\"a\",\"dim5\":\"ab\"}\n{\"t\":\"2000-01-03\",\"m1\":\"3.0\",\"m2\":\"3.0\",\"d1\":0.0,\"d2\":0.0,\"f1\":0.0,\"f2\":0.0,\"l1\":0,\"l2\":0,\"dim1\":\"2\",\"dim2\":[\"\"],\"dim3\":[\"d\"],\"dim4\":\"a\",\"dim5\":\"ba\"}\n{\"t\":\"2001-01-01\",\"m1\":\"4.0\",\"m2\":\"4.0\",\"dim1\":\"1\",\"dim2\":[\"a\"],\"dim3\":[\"\"],\"dim4\":\"b\",\"dim5\":\"ad\"}\n{\"t\":\"2001-01-02\",\"m1\":\"5.0\",\"m2\":\"5.0\",\"dim1\":\"def\",\"dim2\":[\"abc\"],\"dim3\":[],\"dim4\":\"b\",\"dim5\":\"aa\"}\n{\"t\":\"2001-01-03\",\"m1\":\"6.0\",\"m2\":\"6.0\",\"dim1\":\"abc\",\"dim4\":\"b\",\"dim5\":\"ab\"}"
},
"inputFormat": {
"type": "json"
}
},
"tuningConfig": {
"type": "index_parallel",
"partitionsSpec": {
"type": "dynamic"
}
},
"dataSchema": {
"dataSource": "numFoo",
"granularitySpec": {
"type": "uniform",
"queryGranularity": "NONE",
"rollup": false,
"segmentGranularity": "YEAR"
},
"timestampSpec": {
"column": "t",
"format": "auto"
},
"dimensionsSpec": {
"dimensions": [
{
"type": "double",
"name": "d1"
},
{
"type": "double",
"name": "d2"
},
"dim1",
"dim2",
"dim3",
"dim4",
"dim5",
{
"type": "float",
"name": "f1"
},
{
"type": "float",
"name": "f2"
},
{
"type": "long",
"name": "l1"
},
{
"type": "long",
"name": "l2"
}
]
},
"metricsSpec": [
{
"name": "m1",
"type": "floatSum",
"fieldName": "m1"
},
{
"name": "m2",
"type": "doubleSum",
"fieldName": "m2"
}
]
}
}
}