Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow non literal rhs in MV_FILTER_ONLY and MV_FILTER_NONE (#16113) 

This commit allows to use the MV_FILTER_ONLY & MV_FILTER_NONE functions
with a non literal argument. 
Currently `select mv_filter_only('mvd_dim', 'array_dim') from 'table'`
returns a `Unhandled Query Planning Failure`

This is being tackled and also considered for the cases where the `array_dim`
having null & empty values.

Changed classes:
 * `MultiValueStringOperatorConversions`
 * `ApplyFunction`
 * `CalciteMultiValueStringQueryTest`
This commit is contained in:
Sree Charan Manamala 2024-03-26 12:31:09 +05:30 committed by GitHub
parent 95595ba4f5
commit f29c8ac368
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 90 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
processing/src/main/java/org/apache/druid/math/expr/ApplyFunction.java
View File

@ -442,11 +442,15 @@ public interface ApplyFunction extends NamedFunction
Object[] array = arrayEval.asArray();
if (array == null) {
return ExprEval.of(null);
return ExprEval.ofArray(arrayEval.asArrayType(), null);
}
SettableLambdaBinding lambdaBinding = new SettableLambdaBinding(arrayEval.elementType(), lambdaExpr, bindings);
Object[] filtered = filter(arrayEval.asArray(), lambdaExpr, lambdaBinding).toArray();
// return null array expr if nothing is left in filtered
if (filtered.length == 0) {
return ExprEval.ofArray(arrayEval.asArrayType(), null);
}
return ExprEval.ofArray(arrayEval.asArrayType(), filtered);
}

4
processing/src/test/java/org/apache/druid/math/expr/ApplyFunctionTest.java
View File

@ -93,6 +93,10 @@ public class ApplyFunctionTest extends InitializedNullHandlingTest
assertExpr("filter((x) -> x > 2, [1, 2, 3, 4, 5])", new Long[] {3L, 4L, 5L});
assertExpr("filter((x) -> x > 2, b)", new Long[] {3L, 4L, 5L});
String dummyNull = null;
assertExpr("filter((x) -> array_contains([], x), ['a', 'b'])", dummyNull);
assertExpr("filter((x) -> array_contains([], x), null)", dummyNull);
}
@Test

26
sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringOperatorConversions.java
View File

@ -461,20 +461,6 @@ public class MultiValueStringOperatorConversions
return null;
}
Expr expr = plannerContext.parseExpression(druidExpressions.get(1).getExpression());
// the right expression must be a literal array for this to work, since we need the values of the column
if (!expr.isLiteral()) {
return null;
}
Object[] lit = expr.eval(InputBindings.nilBindings()).asArray();
if (lit == null || lit.length == 0) {
return null;
}
HashSet<String> literals = Sets.newHashSetWithExpectedSize(lit.length);
for (Object o : lit) {
literals.add(Evals.asString(o));
}
final DruidExpression.ExpressionGenerator builder = (args) -> {
final StringBuilder expressionBuilder;
if (isAllowList()) {
@ -490,7 +476,17 @@ public class MultiValueStringOperatorConversions
return expressionBuilder.toString();
};
if (druidExpressions.get(0).isSimpleExtraction()) {
Expr expr = plannerContext.parseExpression(druidExpressions.get(1).getExpression());
if (druidExpressions.get(0).isSimpleExtraction() && expr.isLiteral()) {
Object[] lit = expr.eval(InputBindings.nilBindings()).asArray();
if (lit == null || lit.length == 0) {
return null;
}
HashSet<String> literals = Sets.newHashSetWithExpectedSize(lit.length);
for (Object o : lit) {
literals.add(Evals.asString(o));
}
DruidExpression druidExpression = DruidExpression.ofVirtualColumn(
Calcites.getColumnTypeForRelDataType(rexNode.getType()),
builder,

70
sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java
View File

@ -1336,6 +1336,41 @@ public class CalciteMultiValueStringQueryTest extends BaseCalciteQueryTest
);
}
@Test
public void testMultiValueListFilterNonLiteral()
{
// Cannot vectorize due to usage of expressions.
cannotVectorize();
testQuery(
"SELECT MV_FILTER_ONLY(dim3, ARRAY[dim2]) FROM druid.numfoo",
ImmutableList.of(
newScanQueryBuilder()
.dataSource(CalciteTests.DATASOURCE3)
.intervals(querySegmentSpec(Filtration.eternity()))
.virtualColumns(
new ExpressionVirtualColumn(
"v0",
"filter((x) -> array_contains(array(\"dim2\"), x), \"dim3\")",
ColumnType.STRING,
TestExprMacroTable.INSTANCE
)
)
.columns("v0")
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(
new Object[]{"a"},
new Object[]{NullHandling.defaultStringValue()},
new Object[]{NullHandling.defaultStringValue()},
new Object[]{NullHandling.defaultStringValue()},
new Object[]{NullHandling.defaultStringValue()},
new Object[]{NullHandling.defaultStringValue()}
)
);
}
@Test
public void testMultiValueListFilterDeny()
{
@ -1391,6 +1426,41 @@ public class CalciteMultiValueStringQueryTest extends BaseCalciteQueryTest
);
}
@Test
public void testMultiValueListFilterDenyNonLiteral()
{
// Cannot vectorize due to usage of expressions.
cannotVectorize();
testQuery(
"SELECT MV_FILTER_NONE(dim3, ARRAY[dim2]) FROM druid.numfoo",
ImmutableList.of(
newScanQueryBuilder()
.dataSource(CalciteTests.DATASOURCE3)
.intervals(querySegmentSpec(Filtration.eternity()))
.virtualColumns(
new ExpressionVirtualColumn(
"v0",
"filter((x) -> !array_contains(array(\"dim2\"), x), \"dim3\")",
ColumnType.STRING,
TestExprMacroTable.INSTANCE
)
)
.columns("v0")
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(
new Object[]{"b"},
new Object[]{"[\"b\",\"c\"]"},
new Object[]{"d"},
new Object[]{""},
new Object[]{NullHandling.defaultStringValue()},
new Object[]{NullHandling.defaultStringValue()}
)
);
}
@Test
public void testMultiValueListFilterComposed()
{