Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Unnest dont push down not (#14942) 

* Not pushing down not filters

* New test case

* Updating tests

* Removing a stale comment
This commit is contained in:
Soumyava 2023-09-06 08:57:03 -07:00 committed by GitHub
parent 23308c050d
commit a8fa979115
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 47 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java
View File

@ -495,6 +495,8 @@ public class UnnestStorageAdapter implements StorageAdapter
false
);
preFilterList.add(new OrFilter(orChildFilters));
} else if (filter instanceof NotFilter) {
continue;
} else {
final Filter newFilter = rewriteFilterOnUnnestColumnIfPossible(
filter,
@ -579,7 +581,7 @@ public class UnnestStorageAdapter implements StorageAdapter
}
return true;
} else if (filter instanceof NotFilter) {
return filterMapsOverMultiValueStrings(((NotFilter) filter).getBaseFilter());
return false;
} else {
return filter instanceof SelectorFilter
|| filter instanceof InDimFilter

46
processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java
View File

@ -59,6 +59,7 @@ import javax.annotation.Nullable;
import java.util.Arrays;
import java.util.List;
import static org.apache.druid.segment.filter.FilterTestUtils.not;
import static org.apache.druid.segment.filter.FilterTestUtils.selector;
import static org.apache.druid.segment.filter.Filters.and;
import static org.apache.druid.segment.filter.Filters.or;
@ -467,15 +468,54 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
public void test_nested_filters_unnested_and_topLevelAND2sdf()
{
final Filter testQueryFilter = and(ImmutableList.of(
selector(OUTPUT_COLUMN_NAME, "3"),
not(selector(OUTPUT_COLUMN_NAME, "3")),
selector(COLUMNNAME, "2")
));
testComputeBaseAndPostUnnestFilters(
testQueryFilter,
"(multi-string1 = 3 && multi-string1 = 2)",
"(unnested-multi-string1 = 3 && multi-string1 = 2)"
"multi-string1 = 2",
"(~(unnested-multi-string1 = 3) && multi-string1 = 2)"
);
}
@Test
public void test_nested_filters_unnested_and_topLevelOR2sdf()
{
final Filter testQueryFilter = or(ImmutableList.of(
not(selector(OUTPUT_COLUMN_NAME, "3")),
selector(COLUMNNAME, "2")
));
testComputeBaseAndPostUnnestFilters(
testQueryFilter,
"(multi-string1 = 2)",
"(~(unnested-multi-string1 = 3) || multi-string1 = 2)"
);
}
@Test
public void test_not_pushdown_not_filter()
{
final Filter testQueryFilter = and(ImmutableList.of(
not(selector(OUTPUT_COLUMN_NAME, "3")),
or(ImmutableList.of(
or(ImmutableList.of(
selector("newcol", "2"),
selector(COLUMNNAME, "2"),
and(ImmutableList.of(
selector("newcol", "3"),
selector(COLUMNNAME, "7")
))
)),
selector(OUTPUT_COLUMN_NAME, "1")
))
));
testComputeBaseAndPostUnnestFilters(
testQueryFilter,
"(newcol = 2 || multi-string1 = 2 || (newcol = 3 && multi-string1 = 7) || multi-string1 = 1)",
"(~(unnested-multi-string1 = 3) && (newcol = 2 || multi-string1 = 2 || (newcol = 3 && multi-string1 = 7) || unnested-multi-string1 = 1))"
);
}
@Test
public void test_pushdown_filters_unnested_dimension_with_unnest_adapters()
{

13
sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java
View File

@ -3686,19 +3686,8 @@ public class CalciteArraysQueryTest extends BaseCalciteQueryTest
.columns(ImmutableList.of("j0.unnest"))
.build()
),
// The result as incorrect because of pushing the not filter
// into the base. I would expect this result to be 'a'
// Putting the not on the direct mapped column from unnest
// makes it eatup the values from MVD.
// For example select dim3 from numFoo where dim2='a' AND dim1 <> 'foo'
// Has 2 rows
// ["a","b"]
// empty
// if dim3 <> 'b' is pushed down to base it eats up the first row and the
// result only has empty.
// Future developer should ensure not filters involving direct mapping of unnested
// column should not be pushed to base but should onluy appear in the post filter
ImmutableList.of(
new Object[]{"a"},
new Object[]{""}
)
);