Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add new filters to unnest filter pushdown (#14777)

This commit is contained in:
Clint Wylie 2023-08-08 03:29:18 -07:00 committed by GitHub
parent d0403f00fd
commit 2845b6a424
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 68 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java
View File

@ -19,6 +19,7 @@
package org.apache.druid.segment;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import org.apache.druid.java.util.common.Pair;
@ -28,8 +29,11 @@ import org.apache.druid.java.util.common.guava.Sequences;
import org.apache.druid.query.QueryMetrics;
import org.apache.druid.query.filter.BooleanFilter;
import org.apache.druid.query.filter.DimFilter;
import org.apache.druid.query.filter.EqualityFilter;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.InDimFilter;
import org.apache.druid.query.filter.NullFilter;
import org.apache.druid.query.filter.RangeFilter;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.data.Indexed;
@ -452,7 +456,8 @@ public class UnnestStorageAdapter implements StorageAdapter
* over multi-value strings. (Rather than treat them as arrays.) There isn't a method on the Filter interface that
* tells us this, so resort to instanceof.
*/
private static boolean filterMapsOverMultiValueStrings(final Filter filter)
@VisibleForTesting
static boolean filterMapsOverMultiValueStrings(final Filter filter)
{
if (filter instanceof BooleanFilter) {
for (Filter child : ((BooleanFilter) filter).getFilters()) {
@ -468,7 +473,10 @@ public class UnnestStorageAdapter implements StorageAdapter
return filter instanceof SelectorFilter
|| filter instanceof InDimFilter
|| filter instanceof LikeFilter
|| filter instanceof BoundFilter;
|| filter instanceof BoundFilter
|| filter instanceof NullFilter
|| filter instanceof EqualityFilter
|| filter instanceof RangeFilter;
}
}

58
processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java
View File

@ -20,6 +20,7 @@
package org.apache.druid.segment;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.granularity.Granularity;
@ -28,11 +29,26 @@ import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.math.expr.ExprMacroTable;
import org.apache.druid.query.QueryMetrics;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
import org.apache.druid.query.filter.BoundDimFilter;
import org.apache.druid.query.filter.EqualityFilter;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.InDimFilter;
import org.apache.druid.query.filter.LikeDimFilter;
import org.apache.druid.query.filter.NullFilter;
import org.apache.druid.query.filter.RangeFilter;
import org.apache.druid.query.filter.SelectorDimFilter;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.filter.AndFilter;
import org.apache.druid.segment.filter.BoundFilter;
import org.apache.druid.segment.filter.ColumnComparisonFilter;
import org.apache.druid.segment.filter.FalseFilter;
import org.apache.druid.segment.filter.LikeFilter;
import org.apache.druid.segment.filter.NotFilter;
import org.apache.druid.segment.filter.OrFilter;
import org.apache.druid.segment.filter.SelectorFilter;
import org.apache.druid.segment.filter.TrueFilter;
import org.apache.druid.segment.generator.GeneratorBasicSchemas;
import org.apache.druid.segment.generator.GeneratorSchemaInfo;
import org.apache.druid.segment.generator.SegmentGenerator;
@ -392,6 +408,48 @@ public class UnnestStorageAdapterTest extends InitializedNullHandlingTest
return null;
});
}
@Test
public void testAllowedFiltersForPushdown()
{
Filter[] allowed = new Filter[] {
new SelectorFilter("column", "value"),
new InDimFilter("column", ImmutableSet.of("a", "b")),
new LikeFilter("column", null, LikeDimFilter.LikeMatcher.from("hello%", null), null),
new BoundFilter(
new BoundDimFilter("column", "a", "b", true, true, null, null, null)
),
NullFilter.forColumn("column"),
new EqualityFilter("column", ColumnType.LONG, 1234L, null),
new RangeFilter("column", ColumnType.LONG, 0L, 1234L, true, false, null)
};
// not exhaustive
Filter[] notAllowed = new Filter[] {
TrueFilter.instance(),
FalseFilter.instance(),
new ColumnComparisonFilter(ImmutableList.of(DefaultDimensionSpec.of("col1"), DefaultDimensionSpec.of("col2")))
};
for (Filter f : allowed) {
Assert.assertTrue(UnnestStorageAdapter.filterMapsOverMultiValueStrings(f));
}
for (Filter f : notAllowed) {
Assert.assertFalse(UnnestStorageAdapter.filterMapsOverMultiValueStrings(f));
}
Filter notAnd = new NotFilter(
new AndFilter(
Arrays.asList(allowed)
)
);
Assert.assertTrue(UnnestStorageAdapter.filterMapsOverMultiValueStrings(notAnd));
Assert.assertTrue(UnnestStorageAdapter.filterMapsOverMultiValueStrings(new OrFilter(Arrays.asList(allowed))));
Assert.assertTrue(UnnestStorageAdapter.filterMapsOverMultiValueStrings(new NotFilter(notAnd)));
Assert.assertFalse(
UnnestStorageAdapter.filterMapsOverMultiValueStrings(new NotFilter(new OrFilter(Arrays.asList(notAllowed))))
);
}
}
/**