add druid.expressions.allowVectorizeFallback and default to false (#17248) (#17260)

changes:

adds ExpressionProcessing.allowVectorizeFallback() and ExpressionProcessingConfig.allowVectorizeFallback(), defaulting to false until few remaining bugs can be fixed (mostly complex types and some odd interactions with mixed types)
add cannotVectorizeUnlessFallback functions to make it easy to toggle the default of this config, and easy to know what to delete when we remove it in the future

Co-authored-by: Clint Wylie <cwylie@apache.org>
This commit is contained in:
Abhishek Agarwal 2024-10-06 12:29:16 +05:30 committed by GitHub
parent 7b3fc4e768
commit 52441c005c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
24 changed files with 221 additions and 41 deletions

View File

@ -2213,7 +2213,6 @@ Supported query contexts:
|Key|Description|Default|
|---|-----------|-------|
|`druid.expressions.useStrictBooleans`|Controls the behavior of Druid boolean operators and functions, if set to `true` all boolean values are either `1` or `0`. This configuration has been deprecated and will be removed in a future release, taking on the `true` behavior. See [expression documentation](../querying/math-expr.md#logical-operator-modes) for more information.|true|
|`druid.expressions.allowNestedArrays`|If enabled, Druid array expressions can create nested arrays. This configuration has been deprecated and will be removed in a future release, taking on the `true` behavior.|true|
### Router

View File

@ -322,6 +322,7 @@ public class HllSketchSqlAggregatorTest extends BaseCalciteQueryTest
@Test
public void testApproxCountDistinctHllSketch()
{
cannotVectorizeUnlessFallback();
final String sql = "SELECT\n"
+ " SUM(cnt),\n"
+ " APPROX_COUNT_DISTINCT_DS_HLL(dim2),\n" // uppercase
@ -1138,6 +1139,7 @@ public class HllSketchSqlAggregatorTest extends BaseCalciteQueryTest
@Test
public void testHllEstimateAsVirtualColumnWithGroupByOrderBy()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT"
+ " HLL_SKETCH_ESTIMATE(hllsketch_dim1), count(*)"

View File

@ -177,6 +177,7 @@ public class ThetaSketchSqlAggregatorTest extends BaseCalciteQueryTest
@Test
public void testApproxCountDistinctThetaSketch()
{
cannotVectorizeUnlessFallback();
final String sql = "SELECT\n"
+ " SUM(cnt),\n"
+ " APPROX_COUNT_DISTINCT_DS_THETA(dim2),\n"
@ -1157,6 +1158,7 @@ public class ThetaSketchSqlAggregatorTest extends BaseCalciteQueryTest
@Test
public void testThetaEstimateAsVirtualColumnWithGroupByOrderBy()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT"
+ " THETA_SKETCH_ESTIMATE(thetasketch_dim1), count(*)"

View File

@ -182,6 +182,13 @@ public interface Expr extends Cacheable
return false;
}
default boolean canFallbackVectorize(InputBindingInspector inspector, List<Expr> args)
{
return ExpressionProcessing.allowVectorizeFallback() &&
getOutputType(inspector) != null &&
inspector.canVectorize(args);
}
/**
* Possibly convert the {@link Expr} into an optimized, possibly not thread-safe {@link Expr}. Does not convert
* child {@link Expr}. Most callers should use {@link Expr#singleThreaded(Expr, InputBindingInspector)} to convert

View File

@ -178,7 +178,7 @@ public class ExprMacroTable
@Override
public boolean canVectorize(InputBindingInspector inspector)
{
return getOutputType(inspector) != null && inspector.canVectorize(args);
return canFallbackVectorize(inspector, args);
}
@Override

View File

@ -45,19 +45,25 @@ public class ExpressionProcessing
@VisibleForTesting
public static void initializeForTests()
{
INSTANCE = new ExpressionProcessingConfig(null, null, null);
INSTANCE = new ExpressionProcessingConfig(null, null, null, null);
}
@VisibleForTesting
public static void initializeForStrictBooleansTests(boolean useStrict)
{
INSTANCE = new ExpressionProcessingConfig(useStrict, null, null);
INSTANCE = new ExpressionProcessingConfig(useStrict, null, null, null);
}
@VisibleForTesting
public static void initializeForHomogenizeNullMultiValueStrings()
{
INSTANCE = new ExpressionProcessingConfig(null, null, true);
INSTANCE = new ExpressionProcessingConfig(null, null, true, null);
}
@VisibleForTesting
public static void initializeForFallback()
{
INSTANCE = new ExpressionProcessingConfig(null, null, null, true);
}
/**
@ -90,6 +96,12 @@ public class ExpressionProcessing
return INSTANCE.isHomogenizeNullMultiValueStringArrays();
}
public static boolean allowVectorizeFallback()
{
checkInitialized();
return INSTANCE.allowVectorizeFallback();
}
private static void checkInitialized()
{
// this should only be null in a unit test context, in production this will be injected by the null handling module

View File

@ -37,6 +37,7 @@ public class ExpressionProcessingConfig
// Coerce 'null', '[]', and '[null]' into '[null]' for backwards compat with 0.22 and earlier
public static final String HOMOGENIZE_NULL_MULTIVALUE_STRING_ARRAYS =
"druid.expressions.homogenizeNullMultiValueStringArrays";
public static final String ALLOW_VECTORIZE_FALLBACK = "druid.expressions.allowVectorizeFallback";
@JsonProperty("useStrictBooleans")
private final boolean useStrictBooleans;
@ -47,11 +48,15 @@ public class ExpressionProcessingConfig
@JsonProperty("homogenizeNullMultiValueStringArrays")
private final boolean homogenizeNullMultiValueStringArrays;
@JsonProperty("allowVectorizeFallback")
private final boolean allowVectorizeFallback;
@JsonCreator
public ExpressionProcessingConfig(
@JsonProperty("useStrictBooleans") @Nullable Boolean useStrictBooleans,
@JsonProperty("processArraysAsMultiValueStrings") @Nullable Boolean processArraysAsMultiValueStrings,
@JsonProperty("homogenizeNullMultiValueStringArrays") @Nullable Boolean homogenizeNullMultiValueStringArrays
@JsonProperty("homogenizeNullMultiValueStringArrays") @Nullable Boolean homogenizeNullMultiValueStringArrays,
@JsonProperty("allowVectorizeFallback") @Nullable Boolean allowVectorizeFallback
)
{
this.useStrictBooleans = getWithPropertyFallback(
@ -67,6 +72,10 @@ public class ExpressionProcessingConfig
homogenizeNullMultiValueStringArrays,
HOMOGENIZE_NULL_MULTIVALUE_STRING_ARRAYS
);
this.allowVectorizeFallback = getWithPropertyFallbackFalse(
allowVectorizeFallback,
ALLOW_VECTORIZE_FALLBACK
);
String version = ExpressionProcessingConfig.class.getPackage().getImplementationVersion();
if (version == null || version.contains("SNAPSHOT")) {
version = "latest";
@ -95,6 +104,11 @@ public class ExpressionProcessingConfig
return homogenizeNullMultiValueStringArrays;
}
public boolean allowVectorizeFallback()
{
return allowVectorizeFallback;
}
private static boolean getWithPropertyFallbackFalse(@Nullable Boolean value, String property)
{
return getWithPropertyFallback(value, property, "false");

View File

@ -100,8 +100,7 @@ class FunctionExpr implements Expr
@Override
public boolean canVectorize(InputBindingInspector inspector)
{
return function.canVectorize(inspector, args)
|| (getOutputType(inspector) != null && inspector.canVectorize(args));
return function.canVectorize(inspector, args) || canFallbackVectorize(inspector, args);
}
@Override
@ -226,7 +225,8 @@ class ApplyFunctionExpr implements Expr
@Override
public boolean canVectorize(InputBindingInspector inspector)
{
return canVectorizeNative(inspector) || (getOutputType(inspector) != null && inspector.canVectorize(argsExpr));
return canVectorizeNative(inspector) ||
(canFallbackVectorize(inspector, argsExpr) && lambdaExpr.canVectorize(inspector));
}
@Override

View File

@ -30,6 +30,7 @@ import org.apache.druid.math.expr.vector.ExprVectorProcessor;
import org.apache.druid.query.expression.NestedDataExpressions;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Test;
import javax.annotation.Nullable;
@ -264,11 +265,17 @@ public class VectorExprSanityTest extends InitializedNullHandlingTest
@Test
public void testArrayFns()
{
testExpression("array(s1, s2)", types);
testExpression("array(l1, l2)", types);
testExpression("array(d1, d2)", types);
testExpression("array(l1, d2)", types);
testExpression("array(s1, l2)", types);
try {
ExpressionProcessing.initializeForFallback();
testExpression("array(s1, s2)", types);
testExpression("array(l1, l2)", types);
testExpression("array(d1, d2)", types);
testExpression("array(l1, d2)", types);
testExpression("array(s1, l2)", types);
}
finally {
ExpressionProcessing.initializeForTests();
}
}
@Test
@ -284,6 +291,7 @@ public class VectorExprSanityTest extends InitializedNullHandlingTest
@Test
public void testJsonFns()
{
Assume.assumeTrue(ExpressionProcessing.allowVectorizeFallback());
testExpression("json_object('k1', s1, 'k2', l1)", types);
}

View File

@ -35,6 +35,7 @@ import org.apache.druid.java.util.common.granularity.Granularity;
import org.apache.druid.java.util.common.granularity.PeriodGranularity;
import org.apache.druid.java.util.common.guava.Sequence;
import org.apache.druid.java.util.metrics.StubServiceEmitter;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.Druids;
import org.apache.druid.query.FinalizeResultsQueryRunner;
import org.apache.druid.query.MetricsEmittingQueryRunner;
@ -2269,6 +2270,7 @@ public class TimeseriesQueryRunnerTest extends InitializedNullHandlingTest
@Test
public void testTimeSeriesWithFilteredAggAndExpressionFilteredAgg()
{
cannotVectorizeUnlessFallback();
TimeseriesQuery query = Druids
.newTimeseriesQueryBuilder()
.dataSource(QueryRunnerTestHelper.DATA_SOURCE)
@ -3278,4 +3280,12 @@ public class TimeseriesQueryRunnerTest extends InitializedNullHandlingTest
expectedException.expectMessage("Cannot vectorize!");
}
}
protected void cannotVectorizeUnlessFallback()
{
if (vectorize && !ExpressionProcessing.allowVectorizeFallback()) {
expectedException.expect(RuntimeException.class);
expectedException.expectMessage("Cannot vectorize!");
}
}
}

View File

@ -48,6 +48,7 @@ import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprType;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.math.expr.ExpressionType;
import org.apache.druid.math.expr.Parser;
import org.apache.druid.query.QueryContext;
@ -1197,6 +1198,19 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
}
}
protected void assertFilterMatchesSkipVectorizeUnlessFallback(
final DimFilter filter,
final List<String> expectedRows
)
{
final boolean vectorize = ExpressionProcessing.allowVectorizeFallback();
assertFilterMatches(filter, expectedRows, vectorize);
// test double inverted
if (!StringUtils.toLowerCase(testName).contains("concise")) {
assertFilterMatches(NotDimFilter.of(NotDimFilter.of(filter)), expectedRows, vectorize);
}
}
private void assertFilterMatches(
final DimFilter filter,
final List<String> expectedRows,

View File

@ -1644,7 +1644,7 @@ public class EqualityFilterTests
"5", .. [null], [123L, 345L], null
*/
assertFilterMatches(
assertFilterMatchesSkipVectorizeUnlessFallback(
new EqualityFilter(
"arrayStringAsMvd",
ColumnType.STRING,
@ -1653,7 +1653,7 @@ public class EqualityFilterTests
),
ImmutableList.of("0", "3")
);
assertFilterMatches(
assertFilterMatchesSkipVectorizeUnlessFallback(
NotDimFilter.of(
new EqualityFilter(
"arrayStringAsMvd",
@ -1667,7 +1667,7 @@ public class EqualityFilterTests
: ImmutableList.of("1", "2", "4", "5")
);
assertFilterMatches(
assertFilterMatchesSkipVectorizeUnlessFallback(
new EqualityFilter(
"arrayLongAsMvd",
ColumnType.STRING,
@ -1676,7 +1676,7 @@ public class EqualityFilterTests
),
ImmutableList.of("0", "2")
);
assertFilterMatches(
assertFilterMatchesSkipVectorizeUnlessFallback(
NotDimFilter.of(
new EqualityFilter(
"arrayLongAsMvd",
@ -1690,7 +1690,7 @@ public class EqualityFilterTests
: ImmutableList.of("1", "3", "4", "5")
);
assertFilterMatches(
assertFilterMatchesSkipVectorizeUnlessFallback(
new EqualityFilter(
"arrayDoubleAsMvd",
ColumnType.STRING,
@ -1699,7 +1699,7 @@ public class EqualityFilterTests
),
ImmutableList.of("0", "1")
);
assertFilterMatches(
assertFilterMatchesSkipVectorizeUnlessFallback(
NotDimFilter.of(
new EqualityFilter(
"arrayDoubleAsMvd",
@ -1713,7 +1713,7 @@ public class EqualityFilterTests
: ImmutableList.of("2", "3", "4", "5")
);
assertFilterMatches(
assertFilterMatchesSkipVectorizeUnlessFallback(
new EqualityFilter(
"arrayConstantAsMvd",
ColumnType.STRING,

View File

@ -26,6 +26,7 @@ import org.apache.druid.error.DruidException;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExprMacroTable;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.math.expr.ExpressionType;
import org.apache.druid.math.expr.Parser;
import org.apache.druid.query.expression.TestExprMacroTable;
@ -1151,8 +1152,7 @@ public class ExpressionPlannerTest extends InitializedNullHandlingTest
Assert.assertTrue(
thePlan.is(
ExpressionPlan.Trait.NON_SCALAR_INPUTS,
ExpressionPlan.Trait.NEEDS_APPLIED,
ExpressionPlan.Trait.VECTORIZABLE
ExpressionPlan.Trait.NEEDS_APPLIED
)
);
Assert.assertFalse(
@ -1164,6 +1164,7 @@ public class ExpressionPlannerTest extends InitializedNullHandlingTest
ExpressionPlan.Trait.UNKNOWN_INPUTS
)
);
assertFallbackVectorizable(thePlan);
Assert.assertEquals(
"array_to_string(map((\"multi_dictionary_string\") -> array_append(\"scalar_string\", \"multi_dictionary_string\"), \"multi_dictionary_string\"), ',')",
@ -1251,8 +1252,7 @@ public class ExpressionPlannerTest extends InitializedNullHandlingTest
ExpressionPlan thePlan = plan("array(long1, long2)");
Assert.assertTrue(
thePlan.is(
ExpressionPlan.Trait.NON_SCALAR_OUTPUT,
ExpressionPlan.Trait.VECTORIZABLE
ExpressionPlan.Trait.NON_SCALAR_OUTPUT
)
);
Assert.assertFalse(
@ -1265,6 +1265,7 @@ public class ExpressionPlannerTest extends InitializedNullHandlingTest
ExpressionPlan.Trait.NON_SCALAR_INPUTS
)
);
assertFallbackVectorizable(thePlan);
Assert.assertEquals(ExpressionType.LONG_ARRAY, thePlan.getOutputType());
thePlan = plan("array(long1, double1)");
@ -1341,10 +1342,11 @@ public class ExpressionPlannerTest extends InitializedNullHandlingTest
);
Assert.assertTrue(
thePlan.is(
ExpressionPlan.Trait.SINGLE_INPUT_SCALAR,
ExpressionPlan.Trait.VECTORIZABLE
ExpressionPlan.Trait.SINGLE_INPUT_SCALAR
)
);
assertFallbackVectorizable(thePlan);
Assert.assertEquals(ExpressionType.STRING, thePlan.getOutputType());
ColumnCapabilities inferred = thePlan.inferColumnCapabilities(
ExpressionType.toColumnType(thePlan.getOutputType())
@ -1387,8 +1389,7 @@ public class ExpressionPlannerTest extends InitializedNullHandlingTest
{
Assert.assertTrue(
thePlan.is(
ExpressionPlan.Trait.NON_SCALAR_INPUTS,
ExpressionPlan.Trait.VECTORIZABLE
ExpressionPlan.Trait.NON_SCALAR_INPUTS
)
);
Assert.assertFalse(
@ -1401,6 +1402,7 @@ public class ExpressionPlannerTest extends InitializedNullHandlingTest
ExpressionPlan.Trait.NEEDS_APPLIED
)
);
assertFallbackVectorizable(thePlan);
}
private static void assertArrayInAndOut(ExpressionPlan thePlan)
@ -1408,8 +1410,7 @@ public class ExpressionPlannerTest extends InitializedNullHandlingTest
Assert.assertTrue(
thePlan.is(
ExpressionPlan.Trait.NON_SCALAR_INPUTS,
ExpressionPlan.Trait.NON_SCALAR_OUTPUT,
ExpressionPlan.Trait.VECTORIZABLE
ExpressionPlan.Trait.NON_SCALAR_OUTPUT
)
);
Assert.assertFalse(
@ -1421,6 +1422,25 @@ public class ExpressionPlannerTest extends InitializedNullHandlingTest
ExpressionPlan.Trait.NEEDS_APPLIED
)
);
assertFallbackVectorizable(thePlan);
}
private static void assertFallbackVectorizable(ExpressionPlan thePlan)
{
if (ExpressionProcessing.allowVectorizeFallback()) {
Assert.assertTrue(
thePlan.is(
ExpressionPlan.Trait.VECTORIZABLE
)
);
} else {
Assert.assertFalse(
thePlan.is(
ExpressionPlan.Trait.VECTORIZABLE
)
);
}
}
private static class TestMacroTable extends ExprMacroTable

View File

@ -43,6 +43,7 @@ import org.apache.druid.java.util.common.granularity.Granularity;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.math.expr.Evals;
import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.DataSource;
import org.apache.druid.query.Druids;
import org.apache.druid.query.JoinDataSource;
@ -282,6 +283,7 @@ public class BaseCalciteQueryTest extends CalciteTestBase
final boolean useDefault = NullHandling.replaceWithDefault();
public boolean cannotVectorize = false;
public boolean cannotVectorizeUnlessFallback = false;
public boolean skipVectorize = false;
static {
@ -869,7 +871,9 @@ public class BaseCalciteQueryTest extends CalciteTestBase
protected QueryTestBuilder testBuilder()
{
return new QueryTestBuilder(new CalciteTestConfig())
.cannotVectorize(cannotVectorize)
.cannotVectorize(
cannotVectorize || (!ExpressionProcessing.allowVectorizeFallback() && cannotVectorizeUnlessFallback)
)
.skipVectorize(skipVectorize);
}
@ -1288,6 +1292,11 @@ public class BaseCalciteQueryTest extends CalciteTestBase
cannotVectorize = true;
}
protected void cannotVectorizeUnlessFallback()
{
cannotVectorizeUnlessFallback = true;
}
protected void skipVectorize()
{
skipVectorize = true;

View File

@ -1525,6 +1525,7 @@ public class CalciteArraysQueryTest extends BaseCalciteQueryTest
@Test
public void testArrayLength()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT dim1, ARRAY_LENGTH(dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1, 2 ORDER BY 2 DESC",
ImmutableList.of(
@ -1737,6 +1738,7 @@ public class CalciteArraysQueryTest extends BaseCalciteQueryTest
@Test
public void testArrayPrependAppend()
{
cannotVectorizeUnlessFallback();
ImmutableList<Object[]> results;
if (useDefault) {
results = ImmutableList.of(
@ -1855,6 +1857,7 @@ public class CalciteArraysQueryTest extends BaseCalciteQueryTest
@Test
public void testArrayOffset()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT ARRAY_OFFSET(dim3, 1), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
@ -2176,6 +2179,7 @@ public class CalciteArraysQueryTest extends BaseCalciteQueryTest
@Test
public void testArrayOrdinal()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT ARRAY_ORDINAL(dim3, 2), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
@ -2216,6 +2220,7 @@ public class CalciteArraysQueryTest extends BaseCalciteQueryTest
@Test
public void testArrayOffsetOf()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT ARRAY_OFFSET_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
@ -2262,6 +2267,7 @@ public class CalciteArraysQueryTest extends BaseCalciteQueryTest
@Test
public void testArrayOrdinalOf()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT ARRAY_ORDINAL_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
@ -2309,6 +2315,7 @@ public class CalciteArraysQueryTest extends BaseCalciteQueryTest
@Test
public void testArrayToString()
{
cannotVectorizeUnlessFallback();
ImmutableList<Object[]> results;
if (useDefault) {
results = ImmutableList.of(

View File

@ -4218,6 +4218,7 @@ public class CalciteJoinQueryTest extends BaseCalciteQueryTest
@NotYetSupported(Modes.JOIN_CONDITION_NOT_PUSHED_CONDITION)
public void testSemiJoinWithOuterTimeExtractAggregateWithOrderBy()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT COUNT(DISTINCT dim1), EXTRACT(MONTH FROM __time) FROM druid.foo\n"
+ " WHERE dim2 IN (\n"

View File

@ -1936,6 +1936,9 @@ public class CalciteLookupFunctionQueryTest extends BaseCalciteQueryTest
@Test
public void testDontPullUpLookupWhenUsedByAggregation()
{
if (NullHandling.sqlCompatible()) {
cannotVectorizeUnlessFallback();
}
testQuery(
"SELECT LOOKUP(dim1, 'lookyloo121'), COUNT(LOOKUP(dim1, 'lookyloo121')) FROM druid.foo GROUP BY 1",
ImmutableList.of(

View File

@ -469,6 +469,7 @@ public class CalciteMultiValueStringQueryTest extends BaseCalciteQueryTest
@Test
public void testMultiValueStringLength()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT dim1, MV_LENGTH(dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1, 2 ORDER BY 2 DESC",
ImmutableList.of(
@ -629,6 +630,7 @@ public class CalciteMultiValueStringQueryTest extends BaseCalciteQueryTest
@Test
public void testMultiValueStringPrependAppend()
{
cannotVectorizeUnlessFallback();
ImmutableList<Object[]> results;
if (useDefault) {
results = ImmutableList.of(
@ -812,6 +814,7 @@ public class CalciteMultiValueStringQueryTest extends BaseCalciteQueryTest
@Test
public void testMultiValueStringOffset()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT MV_OFFSET(dim3, 1), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
@ -848,6 +851,7 @@ public class CalciteMultiValueStringQueryTest extends BaseCalciteQueryTest
@Test
public void testMultiValueStringOrdinal()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT MV_ORDINAL(dim3, 2), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
@ -888,6 +892,7 @@ public class CalciteMultiValueStringQueryTest extends BaseCalciteQueryTest
@Test
public void testMultiValueStringOffsetOf()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT MV_OFFSET_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
@ -934,6 +939,7 @@ public class CalciteMultiValueStringQueryTest extends BaseCalciteQueryTest
@Test
public void testMultiValueStringOrdinalOf()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT MV_ORDINAL_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC",
ImmutableList.of(
@ -981,6 +987,7 @@ public class CalciteMultiValueStringQueryTest extends BaseCalciteQueryTest
@Test
public void testMultiValueStringToString()
{
cannotVectorizeUnlessFallback();
ImmutableList<Object[]> results;
if (useDefault) {
results = ImmutableList.of(

View File

@ -2681,6 +2681,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
@Test
public void testGroupByPathSelectorFilterCoalesce()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT "
+ "JSON_VALUE(nest, '$.x'), "
@ -7577,6 +7578,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
@Test
public void testToJsonString()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT TO_JSON_STRING(nester) FROM druid.nested GROUP BY 1",
ImmutableList.of(

View File

@ -589,6 +589,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testSafeDivide()
{
cannotVectorizeUnlessFallback();
final Map<String, Object> context = new HashMap<>(QUERY_CONTEXT_DEFAULT);
testQuery(
@ -2744,6 +2745,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testExactCountDistinctWithFilter()
{
cannotVectorizeUnlessFallback();
msqIncompatible();
final String sqlQuery = "SELECT COUNT(DISTINCT foo.dim1) FILTER(WHERE foo.cnt = 1), SUM(foo.cnt) FROM druid.foo";
@ -2770,6 +2772,9 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testExactCountDistinctWithFilter2()
{
if (NullHandling.sqlCompatible()) {
cannotVectorizeUnlessFallback();
}
final String sqlQuery = "SELECT COUNT(DISTINCT foo.dim1) FILTER(WHERE foo.cnt = 1), SUM(foo.cnt) FROM druid.foo";
testQuery(
@ -3259,6 +3264,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testGroupByCaseWhen()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT\n"
+ " CASE EXTRACT(DAY FROM __time)\n"
@ -3382,6 +3388,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testGroupByCaseWhenOfTripleAnd()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT\n"
+ " CASE WHEN m1 > 1 AND m1 < 5 AND cnt = 1 THEN 'x' ELSE NULL END,"
@ -4493,6 +4500,9 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testCountStarOnCommonTableExpression()
{
if (NullHandling.sqlCompatible()) {
cannotVectorizeUnlessFallback();
}
Druids.TimeseriesQueryBuilder builder =
Druids.newTimeseriesQueryBuilder()
.dataSource(CalciteTests.DATASOURCE1)
@ -4530,6 +4540,9 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testCountStarOnView()
{
if (NullHandling.sqlCompatible()) {
cannotVectorizeUnlessFallback();
}
Druids.TimeseriesQueryBuilder builder =
Druids.newTimeseriesQueryBuilder()
.dataSource(CalciteTests.DATASOURCE1)
@ -4574,6 +4587,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
.aggregators(aggregators(new CountAggregatorFactory("a0")))
.context(QUERY_CONTEXT_DEFAULT);
if (NullHandling.sqlCompatible()) {
cannotVectorizeUnlessFallback();
builder = builder.virtualColumns(
expressionVirtualColumn("v0", "substring(\"dim1\", 0, 1)", ColumnType.STRING)
)
@ -5156,6 +5170,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testFilteredAggregations()
{
cannotVectorizeUnlessFallback();
Druids.TimeseriesQueryBuilder builder =
Druids.newTimeseriesQueryBuilder()
.dataSource(CalciteTests.DATASOURCE1)
@ -5340,6 +5355,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testCaseFilteredAggregationWithGroupBy()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT\n"
+ " cnt,\n"
@ -5419,6 +5435,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testExpressionAggregations()
{
cannotVectorizeUnlessFallback();
final ExprMacroTable macroTable = CalciteTests.createExprMacroTable();
testQuery(
@ -5849,6 +5866,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testInExpression()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT dim1 IN ('abc', 'def', 'ghi'), COUNT(*)\n"
+ "FROM druid.foo\n"
@ -5912,6 +5930,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testInOrIsNullExpression()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT dim1 IN ('abc', 'def', 'ghi') OR dim1 IS NULL, COUNT(*)\n"
+ "FROM druid.foo\n"
@ -5943,6 +5962,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testNotInOrIsNullExpression()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT NOT (dim1 IN ('abc', 'def', 'ghi') OR dim1 IS NULL), COUNT(*)\n"
+ "FROM druid.foo\n"
@ -5974,6 +5994,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testNotInAndIsNotNullExpression()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT dim1 NOT IN ('abc', 'def', 'ghi') AND dim1 IS NOT NULL, COUNT(*)\n"
+ "FROM druid.foo\n"
@ -6005,6 +6026,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testInOrGreaterThanExpression()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT dim1 IN ('abc', 'def', 'ghi') OR dim1 > 'zzz', COUNT(*)\n"
+ "FROM druid.foo\n"
@ -6036,6 +6058,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testNotInAndLessThanExpression()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT dim1 NOT IN ('abc', 'def', 'ghi') AND dim1 < 'zzz', COUNT(*)\n"
+ "FROM druid.foo\n"
@ -6067,6 +6090,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testNotInOrEqualToOneOfThemExpression()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT dim1 NOT IN ('abc', 'def', 'ghi') OR dim1 = 'def', COUNT(*)\n"
+ "FROM druid.foo\n"
@ -7295,6 +7319,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testSumOfExtractionFn()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT SUM(CAST(SUBSTRING(dim1, 1, 10) AS INTEGER)) FROM druid.foo",
ImmutableList.of(
@ -8639,6 +8664,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testCountDistinctOfTrim()
{
cannotVectorizeUnlessFallback();
// Test a couple different syntax variants of TRIM.
testQuery(
"SELECT COUNT(DISTINCT TRIM(BOTH ' ' FROM dim1)) FROM druid.foo WHERE TRIM(dim1) <> ''",
@ -8672,6 +8698,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testSillyQuarters()
{
cannotVectorizeUnlessFallback();
// Like FLOOR(__time TO QUARTER) but silly.
testQuery(
"SELECT CAST((EXTRACT(MONTH FROM __time) - 1 ) / 3 + 1 AS INTEGER) AS quarter, COUNT(*)\n"
@ -8782,6 +8809,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testRegexpExtractFilterViaNotNullCheck()
{
cannotVectorizeUnlessFallback();
Druids.TimeseriesQueryBuilder builder =
Druids.newTimeseriesQueryBuilder()
.dataSource(CalciteTests.DATASOURCE1)
@ -9216,6 +9244,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testFilterOnTimeExtract()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT COUNT(*) FROM druid.foo\n"
+ "WHERE EXTRACT(YEAR FROM __time) = 2000\n"
@ -9248,6 +9277,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testFilterOnTimeExtractWithMultipleDays()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT COUNT(*) FROM druid.foo\n"
+ "WHERE EXTRACT(YEAR FROM __time) = 2000\n"
@ -9288,6 +9318,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testFilterOnTimeExtractWithVariousTimeUnits()
{
cannotVectorizeUnlessFallback();
msqIncompatible();
testQuery(
"SELECT COUNT(*) FROM druid.foo4\n"
@ -9397,6 +9428,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testQueryWithSelectProjectAndIdentityProjectDoesNotRename()
{
cannotVectorizeUnlessFallback();
msqIncompatible();
testQuery(
PLANNER_CONFIG_NO_HLL.withOverrides(
@ -9674,6 +9706,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testGroupByStringLength()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT CHARACTER_LENGTH(dim1), COUNT(*) FROM druid.foo GROUP BY CHARACTER_LENGTH(dim1)",
ImmutableList.of(
@ -10913,6 +10946,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testGroupByExtractYear()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT\n"
+ " EXTRACT(YEAR FROM __time) AS \"year\",\n"
@ -10961,6 +10995,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testGroupByFormatYearAndMonth()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT\n"
+ " TIME_FORMAt(__time, 'yyyy MM') AS \"year\",\n"
@ -11009,6 +11044,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testGroupByExtractFloorTime()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT\n"
+ "EXTRACT(YEAR FROM FLOOR(__time TO YEAR)) AS \"year\", SUM(cnt)\n"
@ -11041,6 +11077,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testGroupByExtractFloorTimeLosAngeles()
{
cannotVectorizeUnlessFallback();
testQuery(
PLANNER_CONFIG_DEFAULT,
QUERY_CONTEXT_LOS_ANGELES,
@ -14085,6 +14122,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testExpressionCounts()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT\n"
+ " COUNT(reverse(dim2)),\n"
@ -15080,6 +15118,9 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testGreatestFunctionForNumberWithIsNull()
{
if (NullHandling.sqlCompatible()) {
cannotVectorizeUnlessFallback();
}
String query = "SELECT dim1, MAX(GREATEST(l1, l2)) IS NULL FROM druid.numfoo GROUP BY dim1";
List<Object[]> expectedResult;
@ -15145,6 +15186,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testGreatestFunctionForStringWithIsNull()
{
cannotVectorizeUnlessFallback();
msqIncompatible();
String query = "SELECT l1, LATEST(GREATEST(dim1, dim2)) IS NULL FROM druid.numfoo GROUP BY l1";
@ -15328,6 +15370,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testComplexDecodeAgg()
{
cannotVectorizeUnlessFallback();
msqIncompatible();
testQuery(
"SELECT APPROX_COUNT_DISTINCT_BUILTIN(COMPLEX_DECODE_BASE64('hyperUnique',PARSE_JSON(TO_JSON_STRING(unique_dim1)))) from druid.foo",
@ -15361,6 +15404,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testComplexDecodeAggWithCastedTypeName()
{
cannotVectorizeUnlessFallback();
msqIncompatible();
testQuery(
"SELECT "
@ -16397,6 +16441,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testGroupingSetsWithAggregateCase()
{
cannotVectorizeUnlessFallback();
msqIncompatible();
final Map<String, Object> queryContext = ImmutableMap.of(
PlannerConfig.CTX_KEY_USE_APPROXIMATE_COUNT_DISTINCT, false,

View File

@ -650,6 +650,9 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
@Test
public void testSelectDistinctWithCascadeExtractionFilter()
{
if (NullHandling.sqlCompatible()) {
cannotVectorizeUnlessFallback();
}
testQuery(
"SELECT distinct dim1 FROM druid.foo WHERE substring(substring(dim1, 2), 1, 1) = 'e' OR dim2 = 'a'",
ImmutableList.of(
@ -694,6 +697,7 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
@Test
public void testSelectDistinctWithStrlenFilter()
{
cannotVectorizeUnlessFallback();
testQuery(
"SELECT distinct dim1 FROM druid.foo "
+ "WHERE CHARACTER_LENGTH(dim1) = 3 OR CAST(CHARACTER_LENGTH(dim1) AS varchar) = 3",
@ -2016,6 +2020,7 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
@Test
public void testCountDistinctNonApproximateWithFilter()
{
cannotVectorizeUnlessFallback();
testQuery(
PLANNER_CONFIG_DEFAULT.withOverrides(
ImmutableMap.of(
@ -2054,6 +2059,7 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
@Test
public void testCountDistinctNonApproximateWithFilterHaving()
{
cannotVectorizeUnlessFallback();
testQuery(
PLANNER_CONFIG_DEFAULT.withOverrides(
ImmutableMap.of(

View File

@ -456,6 +456,7 @@ public class CalciteSubqueryTest extends BaseCalciteQueryTest
if (!queryContext.containsKey(QueryContexts.MAX_SUBQUERY_BYTES_KEY)) {
cannotVectorize();
}
cannotVectorizeUnlessFallback();
testQuery(
"SELECT TIME_FORMAT(\"date\", 'yyyy-MM'), SUM(x)\n"
+ "FROM (\n"
@ -1611,6 +1612,7 @@ public class CalciteSubqueryTest extends BaseCalciteQueryTest
@ParameterizedTest(name = "{0}")
public void testScalarInArrayToUseHavingFilter(String testName, Map<String, Object> queryContext)
{
cannotVectorizeUnlessFallback();
DimFilter filter = NullHandling.replaceWithDefault()
? new InDimFilter("v0", new HashSet<>(Arrays.asList("1", "17")))
: new TypedInFilter("v0", ColumnType.LONG, null, ImmutableList.of(1, 17), null);

View File

@ -141,6 +141,7 @@ public class CalciteTimeBoundaryQueryTest extends BaseCalciteQueryTest
@Test
public void testMinTimeQueryWithTimeAndExpressionFilters()
{
cannotVectorizeUnlessFallback();
HashMap<String, Object> queryContext = new HashMap<>(QUERY_CONTEXT_DEFAULT);
queryContext.put(QueryContexts.TIME_BOUNDARY_PLANNING_KEY, true);
testQuery(

View File

@ -21,6 +21,7 @@ package org.apache.druid.sql.calcite;
import com.google.common.collect.ImmutableMap;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.QueryContexts;
import org.apache.druid.quidem.DruidQTestInfo;
import org.apache.druid.quidem.ProjectPathUtils;
@ -52,10 +53,16 @@ public class DecoupledExtension implements BeforeEachCallback
}
private static final ImmutableMap<String, Object> CONTEXT_OVERRIDES = ImmutableMap.<String, Object>builder()
.putAll(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT)
.put(PlannerConfig.CTX_NATIVE_QUERY_SQL_PLANNING_MODE, PlannerConfig.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED)
.put(QueryContexts.ENABLE_DEBUG, true)
.build();
.putAll(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT)
.put(
PlannerConfig.CTX_NATIVE_QUERY_SQL_PLANNING_MODE,
PlannerConfig.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
)
.put(
QueryContexts.ENABLE_DEBUG,
true
)
.build();
public QueryTestBuilder testBuilder()
{
@ -93,17 +100,19 @@ public class DecoupledExtension implements BeforeEachCallback
qCaseDir,
testName,
"quidem testcase reason: " + decTestConfig.quidemReason()
);
);
} else {
return null;
}
}
};
QueryTestBuilder builder = new QueryTestBuilder(testConfig)
.cannotVectorize(baseTest.cannotVectorize)
.skipVectorize(baseTest.skipVectorize);
QueryTestBuilder builder = new QueryTestBuilder(testConfig);
return builder;
return builder.cannotVectorize(
baseTest.cannotVectorize ||
(!ExpressionProcessing.allowVectorizeFallback() && baseTest.cannotVectorizeUnlessFallback)
)
.skipVectorize(baseTest.skipVectorize);
}
}