Merge branch 'rename-d1-dbl1' into unnest-relfieldtrimmer-unnestfieldtype

This commit is contained in:
Zoltan Haindrich 2024-11-15 11:23:15 +00:00
commit f9823b0384
3 changed files with 74 additions and 70 deletions

View File

@ -125,13 +125,13 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
{
cannotVectorize();
BloomKFilter expected1 = new BloomKFilter(TEST_NUM_ENTRIES);
BloomKFilter expectedbl1 = new BloomKFilter(TEST_NUM_ENTRIES);
for (InputRow row : TestDataBuilder.ROWS1_WITH_NUMERIC_DIMS) {
String raw = NullHandling.emptyToNullIfNeeded((String) row.getRaw("dim1"));
if (raw == null) {
expected1.addBytes(null, 0, 0);
expectedbl1.addBytes(null, 0, 0);
} else {
expected1.addString(raw);
expectedbl1.addString(raw);
}
}
@ -159,8 +159,8 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
),
ImmutableList.of(
new Object[]{
queryFramework().queryJsonMapper().writeValueAsString(expected1),
queryFramework().queryJsonMapper().writeValueAsString(expected1)
queryFramework().queryJsonMapper().writeValueAsString(expectedbl1),
queryFramework().queryJsonMapper().writeValueAsString(expectedbl1)
}
)
);
@ -171,14 +171,14 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
{
cannotVectorize();
BloomKFilter expected1 = new BloomKFilter(TEST_NUM_ENTRIES);
BloomKFilter expectedbl1 = new BloomKFilter(TEST_NUM_ENTRIES);
BloomKFilter expected2 = new BloomKFilter(TEST_NUM_ENTRIES);
for (InputRow row : TestDataBuilder.ROWS1_WITH_NUMERIC_DIMS) {
String raw = NullHandling.emptyToNullIfNeeded((String) row.getRaw("dim1"));
if (raw == null) {
expected1.addBytes(null, 0, 0);
expectedbl1.addBytes(null, 0, 0);
} else {
expected1.addString(raw);
expectedbl1.addString(raw);
}
List<String> lst = row.getDimension("dim2");
if (lst.size() == 0) {
@ -223,7 +223,7 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
),
ImmutableList.of(
new Object[] {
jsonMapper.writeValueAsString(expected1),
jsonMapper.writeValueAsString(expectedbl1),
jsonMapper.writeValueAsString(expected2)
}
)
@ -235,14 +235,14 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
{
cannotVectorize();
BloomKFilter expected1 = new BloomKFilter(TEST_NUM_ENTRIES);
BloomKFilter expectedbl1 = new BloomKFilter(TEST_NUM_ENTRIES);
for (InputRow row : TestDataBuilder.ROWS1_WITH_NUMERIC_DIMS) {
String raw = NullHandling.emptyToNullIfNeeded((String) row.getRaw("dim1"));
// empty string extractionFn produces null
if (raw == null || "".equals(raw)) {
expected1.addBytes(null, 0, 0);
expectedbl1.addBytes(null, 0, 0);
} else {
expected1.addString(raw.substring(0, 1));
expectedbl1.addString(raw.substring(0, 1));
}
}
@ -272,7 +272,7 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
.build()
),
ImmutableList.of(
new Object[]{queryFramework().queryJsonMapper().writeValueAsString(expected1)}
new Object[]{queryFramework().queryJsonMapper().writeValueAsString(expectedbl1)}
)
);
}
@ -327,17 +327,17 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
public void testBloomFilterAggLongVirtualColumn() throws Exception
{
cannotVectorize();
BloomKFilter expected1 = new BloomKFilter(TEST_NUM_ENTRIES);
BloomKFilter expectedbl1 = new BloomKFilter(TEST_NUM_ENTRIES);
for (InputRow row : TestDataBuilder.ROWS1_WITH_NUMERIC_DIMS) {
Object raw = row.getRaw("l1");
if (raw == null) {
if (NullHandling.replaceWithDefault()) {
expected1.addLong(NullHandling.defaultLongValue());
expectedbl1.addLong(NullHandling.defaultLongValue());
} else {
expected1.addBytes(null, 0, 0);
expectedbl1.addBytes(null, 0, 0);
}
} else {
expected1.addLong(2 * ((Number) raw).longValue());
expectedbl1.addLong(2 * ((Number) raw).longValue());
}
}
@ -371,7 +371,7 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
.build()
),
ImmutableList.of(
new Object[]{queryFramework().queryJsonMapper().writeValueAsString(expected1)}
new Object[]{queryFramework().queryJsonMapper().writeValueAsString(expectedbl1)}
)
);
}
@ -381,17 +381,17 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
{
cannotVectorize();
BloomKFilter expected1 = new BloomKFilter(TEST_NUM_ENTRIES);
BloomKFilter expectedbl1 = new BloomKFilter(TEST_NUM_ENTRIES);
for (InputRow row : TestDataBuilder.ROWS1_WITH_NUMERIC_DIMS) {
Object raw = row.getRaw("f1");
if (raw == null) {
if (NullHandling.replaceWithDefault()) {
expected1.addFloat(NullHandling.defaultFloatValue());
expectedbl1.addFloat(NullHandling.defaultFloatValue());
} else {
expected1.addBytes(null, 0, 0);
expectedbl1.addBytes(null, 0, 0);
}
} else {
expected1.addFloat(2 * ((Number) raw).floatValue());
expectedbl1.addFloat(2 * ((Number) raw).floatValue());
}
}
@ -425,7 +425,7 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
.build()
),
ImmutableList.of(
new Object[]{queryFramework().queryJsonMapper().writeValueAsString(expected1)}
new Object[]{queryFramework().queryJsonMapper().writeValueAsString(expectedbl1)}
)
);
}
@ -435,23 +435,23 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
{
cannotVectorize();
BloomKFilter expected1 = new BloomKFilter(TEST_NUM_ENTRIES);
BloomKFilter expectedbl1 = new BloomKFilter(TEST_NUM_ENTRIES);
for (InputRow row : TestDataBuilder.ROWS1_WITH_NUMERIC_DIMS) {
Object raw = row.getRaw("d1");
Object raw = row.getRaw("dbl1");
if (raw == null) {
if (NullHandling.replaceWithDefault()) {
expected1.addDouble(NullHandling.defaultDoubleValue());
expectedbl1.addDouble(NullHandling.defaultDoubleValue());
} else {
expected1.addBytes(null, 0, 0);
expectedbl1.addBytes(null, 0, 0);
}
} else {
expected1.addDouble(2 * ((Number) raw).doubleValue());
expectedbl1.addDouble(2 * ((Number) raw).doubleValue());
}
}
testQuery(
"SELECT\n"
+ "BLOOM_FILTER(d1 * 2, 1000)\n"
+ "BLOOM_FILTER(dbl1 * 2, 1000)\n"
+ "FROM numfoo",
ImmutableList.of(
Druids.newTimeseriesQueryBuilder()
@ -461,7 +461,7 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
.virtualColumns(
new ExpressionVirtualColumn(
"v0",
"(\"d1\" * 2)",
"(\"dbl1\" * 2)",
ColumnType.DOUBLE,
TestExprMacroTable.INSTANCE
)
@ -479,7 +479,7 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
.build()
),
ImmutableList.of(
new Object[]{queryFramework().queryJsonMapper().writeValueAsString(expected1)}
new Object[]{queryFramework().queryJsonMapper().writeValueAsString(expectedbl1)}
)
);
}
@ -490,7 +490,7 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
// makes empty bloom filters
cannotVectorize();
BloomKFilter expected1 = new BloomKFilter(TEST_NUM_ENTRIES);
BloomKFilter expectedbl1 = new BloomKFilter(TEST_NUM_ENTRIES);
BloomKFilter expected2 = new BloomKFilter(TEST_NUM_ENTRIES);
ObjectMapper jsonMapper = queryFramework().queryJsonMapper();
@ -524,7 +524,7 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
),
ImmutableList.of(
new Object[] {
jsonMapper.writeValueAsString(expected1),
jsonMapper.writeValueAsString(expectedbl1),
jsonMapper.writeValueAsString(expected2)
}
)
@ -537,7 +537,7 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
// makes empty bloom filters
cannotVectorize();
BloomKFilter expected1 = new BloomKFilter(TEST_NUM_ENTRIES);
BloomKFilter expectedbl1 = new BloomKFilter(TEST_NUM_ENTRIES);
BloomKFilter expected2 = new BloomKFilter(TEST_NUM_ENTRIES);
ObjectMapper jsonMapper = queryFramework().queryJsonMapper();
@ -554,7 +554,7 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
.setDimFilter(equality("dim2", "a", ColumnType.STRING))
.setGranularity(Granularities.ALL)
.setVirtualColumns(expressionVirtualColumn("v0", "'a'", ColumnType.STRING))
.setDimensions(new DefaultDimensionSpec("v0", "_d0", ColumnType.STRING))
.setDimensions(new DefaultDimensionSpec("v0", "d0", ColumnType.STRING))
.setAggregatorSpecs(
aggregators(
new FilteredAggregatorFactory(
@ -581,7 +581,7 @@ public class BloomFilterSqlAggregatorTest extends BaseCalciteQueryTest
ImmutableList.of(
new Object[] {
"a",
jsonMapper.writeValueAsString(expected1),
jsonMapper.writeValueAsString(expectedbl1),
jsonMapper.writeValueAsString(expected2)
}
)

View File

@ -111,7 +111,7 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
new DimensionsSpec(
ImmutableList.<DimensionSchema>builder()
.addAll(DimensionsSpec.getDefaultSchemas(ImmutableList.of("dim1", "dim2", "dim3")))
.add(new DoubleDimensionSchema("d1"))
.add(new DoubleDimensionSchema("dbl1"))
.add(new FloatDimensionSchema("f1"))
.add(new LongDimensionSchema("l1"))
.build()
@ -176,7 +176,7 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
VarianceAggregatorCollector holder2 = new VarianceAggregatorCollector();
VarianceAggregatorCollector holder3 = new VarianceAggregatorCollector();
for (InputRow row : TestDataBuilder.ROWS1_WITH_NUMERIC_DIMS) {
Object raw1 = row.getRaw("d1");
Object raw1 = row.getRaw("dbl1");
Object raw2 = row.getRaw("f1");
Object raw3 = row.getRaw("l1");
addToHolder(holder1, raw1);
@ -193,7 +193,7 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
);
testQuery(
"SELECT\n"
+ "VAR_POP(d1),\n"
+ "VAR_POP(dbl1),\n"
+ "VAR_POP(f1),\n"
+ "VAR_POP(l1)\n"
+ "FROM numfoo",
@ -204,7 +204,7 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
.granularity(Granularities.ALL)
.aggregators(
ImmutableList.of(
new VarianceAggregatorFactory("a0", "d1", "population", "double"),
new VarianceAggregatorFactory("a0", "dbl1", "population", "double"),
new VarianceAggregatorFactory("a1", "f1", "population", "float"),
new VarianceAggregatorFactory("a2", "l1", "population", "long")
)
@ -224,7 +224,7 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
VarianceAggregatorCollector holder2 = new VarianceAggregatorCollector();
VarianceAggregatorCollector holder3 = new VarianceAggregatorCollector();
for (InputRow row : TestDataBuilder.ROWS1_WITH_NUMERIC_DIMS) {
Object raw1 = row.getRaw("d1");
Object raw1 = row.getRaw("dbl1");
Object raw2 = row.getRaw("f1");
Object raw3 = row.getRaw("l1");
addToHolder(holder1, raw1);
@ -241,7 +241,7 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
);
testQuery(
"SELECT\n"
+ "VAR_SAMP(d1),\n"
+ "VAR_SAMP(dbl1),\n"
+ "VAR_SAMP(f1),\n"
+ "VAR_SAMP(l1)\n"
+ "FROM numfoo",
@ -252,7 +252,7 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
.granularity(Granularities.ALL)
.aggregators(
ImmutableList.of(
new VarianceAggregatorFactory("a0", "d1", "sample", "double"),
new VarianceAggregatorFactory("a0", "dbl1", "sample", "double"),
new VarianceAggregatorFactory("a1", "f1", "sample", "float"),
new VarianceAggregatorFactory("a2", "l1", "sample", "long")
)
@ -272,7 +272,7 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
VarianceAggregatorCollector holder2 = new VarianceAggregatorCollector();
VarianceAggregatorCollector holder3 = new VarianceAggregatorCollector();
for (InputRow row : TestDataBuilder.ROWS1_WITH_NUMERIC_DIMS) {
Object raw1 = row.getRaw("d1");
Object raw1 = row.getRaw("dbl1");
Object raw2 = row.getRaw("f1");
Object raw3 = row.getRaw("l1");
addToHolder(holder1, raw1);
@ -290,7 +290,7 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
testQuery(
"SELECT\n"
+ "STDDEV_POP(d1),\n"
+ "STDDEV_POP(dbl1),\n"
+ "STDDEV_POP(f1),\n"
+ "STDDEV_POP(l1)\n"
+ "FROM numfoo",
@ -301,7 +301,7 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
.granularity(Granularities.ALL)
.aggregators(
ImmutableList.of(
new VarianceAggregatorFactory("a0:agg", "d1", "population", "double"),
new VarianceAggregatorFactory("a0:agg", "dbl1", "population", "double"),
new VarianceAggregatorFactory("a1:agg", "f1", "population", "float"),
new VarianceAggregatorFactory("a2:agg", "l1", "population", "long")
)
@ -328,7 +328,7 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
VarianceAggregatorCollector holder2 = new VarianceAggregatorCollector();
VarianceAggregatorCollector holder3 = new VarianceAggregatorCollector();
for (InputRow row : TestDataBuilder.ROWS1_WITH_NUMERIC_DIMS) {
Object raw1 = row.getRaw("d1");
Object raw1 = row.getRaw("dbl1");
Object raw2 = row.getRaw("f1");
Object raw3 = row.getRaw("l1");
addToHolder(holder1, raw1);
@ -346,7 +346,7 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
testQuery(
"SELECT\n"
+ "STDDEV_SAMP(d1),\n"
+ "STDDEV_SAMP(dbl1),\n"
+ "STDDEV_SAMP(f1),\n"
+ "STDDEV_SAMP(l1)\n"
+ "FROM numfoo",
@ -357,7 +357,7 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
.granularity(Granularities.ALL)
.aggregators(
ImmutableList.of(
new VarianceAggregatorFactory("a0:agg", "d1", "sample", "double"),
new VarianceAggregatorFactory("a0:agg", "dbl1", "sample", "double"),
new VarianceAggregatorFactory("a1:agg", "f1", "sample", "float"),
new VarianceAggregatorFactory("a2:agg", "l1", "sample", "long")
)
@ -382,7 +382,7 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
VarianceAggregatorCollector holder2 = new VarianceAggregatorCollector();
VarianceAggregatorCollector holder3 = new VarianceAggregatorCollector();
for (InputRow row : TestDataBuilder.ROWS1_WITH_NUMERIC_DIMS) {
Object raw1 = row.getRaw("d1");
Object raw1 = row.getRaw("dbl1");
Object raw2 = row.getRaw("f1");
Object raw3 = row.getRaw("l1");
addToHolder(holder1, raw1, 7);
@ -400,7 +400,7 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
testQuery(
"SELECT\n"
+ "STDDEV(d1*7),\n"
+ "STDDEV(dbl1*7),\n"
+ "STDDEV(f1*7),\n"
+ "STDDEV(l1*7)\n"
+ "FROM numfoo",
@ -410,7 +410,7 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
.intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
.granularity(Granularities.ALL)
.virtualColumns(
BaseCalciteQueryTest.expressionVirtualColumn("v0", "(\"d1\" * 7)", ColumnType.DOUBLE),
BaseCalciteQueryTest.expressionVirtualColumn("v0", "(\"dbl1\" * 7)", ColumnType.DOUBLE),
BaseCalciteQueryTest.expressionVirtualColumn("v1", "(\"f1\" * 7)", ColumnType.FLOAT),
BaseCalciteQueryTest.expressionVirtualColumn("v2", "(\"l1\" * 7)", ColumnType.LONG)
)
@ -457,7 +457,7 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
.setDataSource(CalciteTests.DATASOURCE3)
.setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
.setGranularity(Granularities.ALL)
.setDimensions(new DefaultDimensionSpec("dim2", "_d0"))
.setDimensions(new DefaultDimensionSpec("dim2", "d0"))
.setAggregatorSpecs(
new VarianceAggregatorFactory("a0", "f1", "sample", "float")
)
@ -527,10 +527,10 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
{
testQuery(
"SELECT\n"
+ "STDDEV_POP(d1),\n"
+ "STDDEV_SAMP(d1),\n"
+ "STDDEV(d1),\n"
+ "VARIANCE(d1),\n"
+ "STDDEV_POP(dbl1),\n"
+ "STDDEV_SAMP(dbl1),\n"
+ "STDDEV(dbl1),\n"
+ "VARIANCE(dbl1),\n"
+ "STDDEV_POP(l1),\n"
+ "STDDEV_SAMP(l1),\n"
+ "STDDEV(l1),\n"
@ -543,10 +543,10 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
.granularity(Granularities.ALL)
.filters(numericEquality("dim2", 0L, ColumnType.LONG))
.aggregators(
new VarianceAggregatorFactory("a0:agg", "d1", "population", "double"),
new VarianceAggregatorFactory("a1:agg", "d1", "sample", "double"),
new VarianceAggregatorFactory("a2:agg", "d1", "sample", "double"),
new VarianceAggregatorFactory("a3", "d1", "sample", "double"),
new VarianceAggregatorFactory("a0:agg", "dbl1", "population", "double"),
new VarianceAggregatorFactory("a1:agg", "dbl1", "sample", "double"),
new VarianceAggregatorFactory("a2:agg", "dbl1", "sample", "double"),
new VarianceAggregatorFactory("a3", "dbl1", "sample", "double"),
new VarianceAggregatorFactory("a4:agg", "l1", "population", "long"),
new VarianceAggregatorFactory("a5:agg", "l1", "sample", "long"),
new VarianceAggregatorFactory("a6:agg", "l1", "sample", "long"),
@ -579,10 +579,10 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
testQuery(
"SELECT\n"
+ "dim2,\n"
+ "STDDEV_POP(d1) FILTER(WHERE dim1 = 'nonexistent'),\n"
+ "STDDEV_SAMP(d1) FILTER(WHERE dim1 = 'nonexistent'),\n"
+ "STDDEV(d1) FILTER(WHERE dim1 = 'nonexistent'),\n"
+ "VARIANCE(d1) FILTER(WHERE dim1 = 'nonexistent'),\n"
+ "STDDEV_POP(dbl1) FILTER(WHERE dim1 = 'nonexistent'),\n"
+ "STDDEV_SAMP(dbl1) FILTER(WHERE dim1 = 'nonexistent'),\n"
+ "STDDEV(dbl1) FILTER(WHERE dim1 = 'nonexistent'),\n"
+ "VARIANCE(dbl1) FILTER(WHERE dim1 = 'nonexistent'),\n"
+ "STDDEV_POP(l1) FILTER(WHERE dim1 = 'nonexistent'),\n"
+ "STDDEV_SAMP(l1) FILTER(WHERE dim1 = 'nonexistent'),\n"
+ "STDDEV(l1) FILTER(WHERE dim1 = 'nonexistent'),\n"
@ -595,23 +595,23 @@ public class VarianceSqlAggregatorTest extends BaseCalciteQueryTest
.setDimFilter(equality("dim2", "a", ColumnType.STRING))
.setGranularity(Granularities.ALL)
.setVirtualColumns(expressionVirtualColumn("v0", "'a'", ColumnType.STRING))
.setDimensions(new DefaultDimensionSpec("v0", "_d0", ColumnType.STRING))
.setDimensions(new DefaultDimensionSpec("v0", "d0", ColumnType.STRING))
.setAggregatorSpecs(
aggregators(
new FilteredAggregatorFactory(
new VarianceAggregatorFactory("a0:agg", "d1", "population", "double"),
new VarianceAggregatorFactory("a0:agg", "dbl1", "population", "double"),
equality("dim1", "nonexistent", ColumnType.STRING)
),
new FilteredAggregatorFactory(
new VarianceAggregatorFactory("a1:agg", "d1", "sample", "double"),
new VarianceAggregatorFactory("a1:agg", "dbl1", "sample", "double"),
equality("dim1", "nonexistent", ColumnType.STRING)
),
new FilteredAggregatorFactory(
new VarianceAggregatorFactory("a2:agg", "d1", "sample", "double"),
new VarianceAggregatorFactory("a2:agg", "dbl1", "sample", "double"),
equality("dim1", "nonexistent", ColumnType.STRING)
),
new FilteredAggregatorFactory(
new VarianceAggregatorFactory("a3", "d1", "sample", "double"),
new VarianceAggregatorFactory("a3", "dbl1", "sample", "double"),
equality("dim1", "nonexistent", ColumnType.STRING)
),
new FilteredAggregatorFactory(

View File

@ -79,6 +79,7 @@ import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.fail;
import static org.junit.Assume.assumeTrue;
/**
* These test cases are borrowed from the drill-test-framework at
@ -113,9 +114,12 @@ public class DrillWindowQueryTest extends BaseCalciteQueryTest
@Test
public void ensureAllDeclared() throws Exception
{
assumeTrue(DrillWindowQueryTest.class.equals(getClass()));
final URL windowQueriesUrl = ClassLoader.getSystemResource("drill/window/queries/");
Path windowFolder = new File(windowQueriesUrl.toURI()).toPath();
Set<String> allCases = FileUtils
.streamFiles(windowFolder.toFile(), true, "q")
.map(file -> {