Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix SQL queries for inline datasource with null values (#12092) 

Fixes a bug because of which some SQL queries cannot be parsed using druid convention. Specifically, these queries translate to an inline datasource and have some null values. Calcite internally uses NULL as SQL type for these literals and that is not supported by the druid.
I am now allowing null column types to be returned while building RowSignature in org.apache.druid.sql.calcite.table.RowSignatures#fromRelDataType. RowSignature already allows null column type for any column. Doing so should also fix bindable queries such as select (1,2). When such queries are run with headers set to true, we get an exception in org.apache.druid.sql.http.ArrayWriter#writeHeader. This is again a similar exception to the one addressed in this PR. Because SQL type for the result column is RECORD and that doesn't have a corresponding columnType.
This commit is contained in:
Abhishek Agarwal 2022-01-27 18:04:12 +05:30 committed by GitHub
parent a813816fb1
commit 1b8808cce8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 199 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java
View File

@ -242,6 +242,7 @@ public class Expressions
.lookupOperatorConversion(operator);
if (conversion == null) {
plannerContext.setPlanningError("SQL query requires '%s' operator that is not supported.", operator.getName());
return null;
} else {

5
sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRule.java
View File

@ -128,6 +128,11 @@ public class DruidLogicalValuesRule extends RelOptRule
case TIMESTAMP:
case DATE:
return Calcites.calciteDateTimeLiteralToJoda(literal, plannerContext.getTimeZone()).getMillis();
case NULL:
if (!literal.isNull()) {
throw new UnsupportedSQLQueryException("Query has a non-null constant but is of NULL type.");
}
return null;
case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
case TIME:
case TIME_WITH_LOCAL_TIME_ZONE:

8
sql/src/main/java/org/apache/druid/sql/calcite/table/RowSignatures.java
View File

@ -61,14 +61,6 @@ public class RowSignatures
for (int i = 0; i < rowOrder.size(); i++) {
final RelDataType dataType = rowType.getFieldList().get(i).getType();
final ColumnType valueType = Calcites.getColumnTypeForRelDataType(dataType);
if (valueType == null) {
throw new ISE(
"Cannot translate sqlTypeName[%s] to Druid type for field[%s]",
dataType.getSqlTypeName(),
rowOrder.get(i)
);
}
rowSignatureBuilder.add(rowOrder.get(i), valueType);
}

3
sql/src/main/java/org/apache/druid/sql/http/ArrayWriter.java
View File

@ -23,6 +23,7 @@ import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.column.TypeSignature;
import org.apache.druid.sql.calcite.table.RowSignatures;
import javax.annotation.Nullable;
@ -111,7 +112,7 @@ public class ArrayWriter implements ResultFormat.Writer
if (includeTypes) {
jsonGenerator.writeStartArray();
for (int i = 0; i < signature.size(); i++) {
jsonGenerator.writeString(signature.getColumnType(i).get().asTypeString());
jsonGenerator.writeString(signature.getColumnType(i).map(TypeSignature::asTypeString).orElse(null));
}
jsonGenerator.writeEndArray();
}

3
sql/src/main/java/org/apache/druid/sql/http/CsvWriter.java
View File

@ -22,6 +22,7 @@ package org.apache.druid.sql.http;
import com.opencsv.CSVWriter;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.column.TypeSignature;
import org.apache.druid.sql.calcite.table.RowSignatures;
import javax.annotation.Nullable;
@ -76,7 +77,7 @@ public class CsvWriter implements ResultFormat.Writer
final String[] types = new String[rowType.getFieldCount()];
for (int i = 0; i < signature.size(); i++) {
types[i] = signature.getColumnType(i).get().asTypeString();
types[i] = signature.getColumnType(i).map(TypeSignature::asTypeString).orElse(null);
}
writer.writeNext(types, false);

3
sql/src/main/java/org/apache/druid/sql/http/ObjectWriter.java
View File

@ -23,6 +23,7 @@ import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.column.TypeSignature;
import org.apache.druid.sql.calcite.table.RowSignatures;
import javax.annotation.Nullable;
@ -119,7 +120,7 @@ public class ObjectWriter implements ResultFormat.Writer
if (includeTypes) {
jsonGenerator.writeStringField(
ObjectWriter.TYPE_HEADER_NAME,
signature.getColumnType(i).get().asTypeString()
signature.getColumnType(i).map(TypeSignature::asTypeString).orElse(null)
);
}

84
sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
View File

@ -122,6 +122,90 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
);
}
@Test
public void testValuesContainingNull() throws Exception
{
testQuery(
"SELECT * FROM (VALUES (NULL, 'United States'))",
ImmutableList.of(
Druids.newScanQueryBuilder()
.dataSource(
InlineDataSource.fromIterable(
ImmutableList.of(new Object[]{null, "United States"}),
RowSignature
.builder()
.add("EXPR$0", null)
.add("EXPR$1", ColumnType.STRING)
.build()
)
)
.intervals(querySegmentSpec(Filtration.eternity()))
.columns("EXPR$0", "EXPR$1")
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.legacy(false)
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(new Object[]{null, "United States"})
);
}
@Test
public void testMultipleValuesContainingNull() throws Exception
{
testQuery(
"SELECT * FROM (VALUES (NULL, 'United States'), ('Delhi', 'India'))",
ImmutableList.of(
Druids.newScanQueryBuilder()
.dataSource(
InlineDataSource.fromIterable(
ImmutableList.of(new Object[]{null, "United States"}, new Object[]{"Delhi", "India"}),
RowSignature
.builder()
.add("EXPR$0", ColumnType.STRING)
.add("EXPR$1", ColumnType.STRING)
.build()
)
)
.intervals(querySegmentSpec(Filtration.eternity()))
.columns("EXPR$0", "EXPR$1")
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.legacy(false)
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(new Object[]{NULL_STRING, "United States"}, new Object[]{"Delhi", "India"})
);
}
@Test
public void testMultipleValuesContainingNullAndIntegerValues() throws Exception
{
testQuery(
"SELECT * FROM (VALUES (NULL, 'United States'), (50, 'India'))",
ImmutableList.of(
Druids.newScanQueryBuilder()
.dataSource(
InlineDataSource.fromIterable(
ImmutableList.of(new Object[]{null, "United States"}, new Object[]{50L, "India"}),
RowSignature
.builder()
.add("EXPR$0", ColumnType.LONG)
.add("EXPR$1", ColumnType.STRING)
.build()
)
)
.intervals(querySegmentSpec(Filtration.eternity()))
.columns("EXPR$0", "EXPR$1")
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.legacy(false)
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(new Object[]{null, "United States"}, new Object[]{50, "India"})
);
}
@Test
public void testSelectNonNumericNumberLiterals() throws Exception
{

103
sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java
View File

@ -100,6 +100,7 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -671,6 +672,30 @@ public class SqlResourceTest extends CalciteTestBase
);
}
@Test
public void testArrayResultFormatWithHeader_nullColumnType() throws Exception
{
// Test a query that returns null header for some of the columns
final String query = "SELECT (1, 2)";
Assert.assertEquals(
ImmutableList.of(
Collections.singletonList("EXPR$0"),
Collections.singletonList(null),
Collections.singletonList("ROW"),
Collections.singletonList(
Arrays.asList(
1,
2
)
)
),
doPost(
new SqlQuery(query, ResultFormat.ARRAY, true, true, true, null, null),
new TypeReference<List<List<Object>>>() {}
).rhs
);
}
@Test
public void testArrayLinesResultFormat() throws Exception
{
@ -764,6 +789,34 @@ public class SqlResourceTest extends CalciteTestBase
Assert.assertEquals("", lines.get(6));
}
@Test
public void testArrayLinesResultFormatWithHeader_nullColumnType() throws Exception
{
final String query = "SELECT (1, 2)";
final Pair<QueryException, String> pair = doPostRaw(
new SqlQuery(query, ResultFormat.ARRAYLINES, true, true, true, null, null)
);
Assert.assertNull(pair.lhs);
final String response = pair.rhs;
final List<String> lines = Splitter.on('\n').splitToList(response);
Assert.assertEquals(6, lines.size());
Assert.assertEquals(Collections.singletonList("EXPR$0"), JSON_MAPPER.readValue(lines.get(0), List.class));
Assert.assertEquals(Collections.singletonList(null), JSON_MAPPER.readValue(lines.get(1), List.class));
Assert.assertEquals(Collections.singletonList("ROW"), JSON_MAPPER.readValue(lines.get(2), List.class));
Assert.assertEquals(
Collections.singletonList(
Arrays.asList(
1,
2
)
),
JSON_MAPPER.readValue(lines.get(3), List.class)
);
Assert.assertEquals("", lines.get(4));
Assert.assertEquals("", lines.get(5));
}
@Test
public void testObjectResultFormat() throws Exception
{
@ -993,6 +1046,35 @@ public class SqlResourceTest extends CalciteTestBase
Assert.assertEquals("", lines.get(4));
}
@Test
public void testObjectLinesResultFormatWithFullHeader_nullColumnType() throws Exception
{
final String query = "SELECT (1, 2)";
final Pair<QueryException, String> pair =
doPostRaw(new SqlQuery(query, ResultFormat.OBJECTLINES, true, true, true, null, null));
Assert.assertNull(pair.lhs);
final String response = pair.rhs;
final List<String> lines = Splitter.on('\n').splitToList(response);
final Map<String, String> typeMap = new HashMap<>();
typeMap.put(ObjectWriter.TYPE_HEADER_NAME, null);
typeMap.put(ObjectWriter.SQL_TYPE_HEADER_NAME, "ROW");
final Map<String, Object> expectedHeader = ImmutableMap.of("EXPR$0", typeMap);
Assert.assertEquals(4, lines.size());
Assert.assertEquals(expectedHeader, JSON_MAPPER.readValue(lines.get(0), Object.class));
Assert.assertEquals(
ImmutableMap
.<String, Object>builder()
.put("EXPR$0", Arrays.asList(1, 2))
.build(),
JSON_MAPPER.readValue(lines.get(1), Object.class)
);
Assert.assertEquals("", lines.get(2));
Assert.assertEquals("", lines.get(3));
}
@Test
public void testCsvResultFormat() throws Exception
{
@ -1040,6 +1122,27 @@ public class SqlResourceTest extends CalciteTestBase
);
}
@Test
public void testCsvResultFormatWithHeaders_nullColumnType() throws Exception
{
final String query = "SELECT (1, 2)";
final Pair<QueryException, String> pair = doPostRaw(
new SqlQuery(query, ResultFormat.CSV, true, true, true, null, null)
);
Assert.assertNull(pair.lhs);
final String response = pair.rhs;
final List<String> lines = Splitter.on('\n').splitToList(response);
Assert.assertEquals(
ImmutableList.of(
"EXPR$0",
"",
"ROW"
),
lines.subList(0, 3)
);
}
@Test
public void testExplainCountStar() throws Exception
{