Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Ability to filter on virtual columns. (#3942) 

This didn't need much other than having BitmapIndexSelector return null from
various methods to trigger cursor based filtering.
This commit is contained in:
Gian Merlino 2017-02-21 16:03:31 -08:00 committed by Jonathan Wei
parent f910c050fd
commit a47206eaf8
8 changed files with 127 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
processing/src/main/java/io/druid/query/search/search/AutoStrategy.java
View File

@ -25,6 +25,7 @@ import io.druid.query.filter.BitmapIndexSelector;
import io.druid.segment.ColumnSelectorBitmapIndexSelector;
import io.druid.segment.QueryableIndex;
import io.druid.segment.Segment;
import io.druid.segment.VirtualColumns;
import io.druid.segment.column.BitmapIndex;
import io.druid.segment.column.Column;
@ -54,6 +55,7 @@ public class AutoStrategy extends SearchStrategy
if (index != null) {
final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(
index.getBitmapFactoryForDimensions(),
VirtualColumns.EMPTY,
index
);

3
processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java
View File

@ -37,6 +37,7 @@ import io.druid.segment.ColumnSelectorBitmapIndexSelector;
import io.druid.segment.QueryableIndex;
import io.druid.segment.Segment;
import io.druid.segment.StorageAdapter;
import io.druid.segment.VirtualColumns;
import io.druid.segment.column.BitmapIndex;
import io.druid.segment.column.Column;
import io.druid.segment.column.ColumnCapabilities;
@ -80,6 +81,7 @@ public class UseIndexesStrategy extends SearchStrategy
if (bitmapSuppDims.size() > 0) {
final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(
index.getBitmapFactoryForDimensions(),
VirtualColumns.EMPTY,
index
);
@ -153,6 +155,7 @@ public class UseIndexesStrategy extends SearchStrategy
} else {
final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(
index.getBitmapFactoryForDimensions(),
VirtualColumns.EMPTY,
index
);
Preconditions.checkArgument(filter.supportsBitmapIndex(selector), "filter[%s] should support bitmap", filter);

33
processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java
View File

@ -26,6 +26,7 @@ import io.druid.collections.spatial.ImmutableRTree;
import io.druid.query.filter.BitmapIndexSelector;
import io.druid.segment.column.BitmapIndex;
import io.druid.segment.column.Column;
import io.druid.segment.column.ColumnCapabilities;
import io.druid.segment.column.DictionaryEncodedColumn;
import io.druid.segment.column.GenericColumn;
import io.druid.segment.column.ValueType;
@ -40,20 +41,28 @@ import java.util.Iterator;
public class ColumnSelectorBitmapIndexSelector implements BitmapIndexSelector
{
private final BitmapFactory bitmapFactory;
private final VirtualColumns virtualColumns;
private final ColumnSelector index;
public ColumnSelectorBitmapIndexSelector(
final BitmapFactory bitmapFactory,
final VirtualColumns virtualColumns,
final ColumnSelector index
)
{
this.bitmapFactory = bitmapFactory;
this.virtualColumns = virtualColumns;
this.index = index;
}
@Override
public Indexed<String> getDimensionValues(String dimension)
{
if (isFilterableVirtualColumn(dimension)) {
// Virtual columns don't have dictionaries or indexes.
return null;
}
final Column columnDesc = index.getColumn(dimension);
if (columnDesc == null || !columnDesc.getCapabilities().isDictionaryEncoded()) {
return null;
@ -110,6 +119,11 @@ public class ColumnSelectorBitmapIndexSelector implements BitmapIndexSelector
@Override
public BitmapIndex getBitmapIndex(String dimension)
{
if (isFilterableVirtualColumn(dimension)) {
// Virtual columns don't have dictionaries or indexes.
return null;
}
final Column column = index.getColumn(dimension);
if (column == null || !columnSupportsFiltering(column)) {
// for missing columns and columns with types that do not support filtering,
@ -172,6 +186,11 @@ public class ColumnSelectorBitmapIndexSelector implements BitmapIndexSelector
@Override
public ImmutableBitmap getBitmapIndex(String dimension, String value)
{
if (isFilterableVirtualColumn(dimension)) {
// Virtual columns don't have dictionaries or indexes.
return null;
}
final Column column = index.getColumn(dimension);
if (column == null || !columnSupportsFiltering(column)) {
if (Strings.isNullOrEmpty(value)) {
@ -192,6 +211,10 @@ public class ColumnSelectorBitmapIndexSelector implements BitmapIndexSelector
@Override
public ImmutableRTree getSpatialIndex(String dimension)
{
if (isFilterableVirtualColumn(dimension)) {
return new ImmutableRTree();
}
final Column column = index.getColumn(dimension);
if (column == null || !column.getCapabilities().hasSpatialIndexes()) {
return new ImmutableRTree();
@ -200,6 +223,16 @@ public class ColumnSelectorBitmapIndexSelector implements BitmapIndexSelector
return column.getSpatialIndex().getRTree();
}
private boolean isFilterableVirtualColumn(final String columnName)
{
final ColumnCapabilities columnCapabilities = virtualColumns.getColumnCapabilities(columnName);
if (columnCapabilities == null) {
return false;
} else {
return Filters.FILTERABLE_TYPES.contains(columnCapabilities.getType());
}
}
private static boolean columnSupportsFiltering(Column column)
{
ValueType columnType = column.getCapabilities().getType();

1
processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java
View File

@ -226,6 +226,7 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
final ColumnSelectorBitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(
index.getBitmapFactoryForDimensions(),
virtualColumns,
index
);

56
processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java
View File

@ -42,13 +42,16 @@ import io.druid.query.extraction.ExtractionFn;
import io.druid.query.extraction.JavaScriptExtractionFn;
import io.druid.query.extraction.MapLookupExtractor;
import io.druid.query.filter.AndDimFilter;
import io.druid.query.filter.BoundDimFilter;
import io.druid.query.filter.DimFilter;
import io.druid.query.filter.SelectorDimFilter;
import io.druid.query.lookup.LookupExtractionFn;
import io.druid.query.ordering.StringComparators;
import io.druid.query.spec.LegacySegmentSpec;
import io.druid.query.spec.QuerySegmentSpec;
import io.druid.segment.column.Column;
import io.druid.segment.column.ValueType;
import io.druid.segment.virtual.ExpressionVirtualColumn;
import org.joda.time.DateTime;
import org.joda.time.Interval;
import org.junit.Assert;
@ -479,6 +482,59 @@ public class SelectQueryRunnerTest
}
}
@Test
public void testFullOnSelectWithFilterOnVirtualColumn()
{
SelectQuery query = newTestQuery()
.intervals("2011-01-13/2011-01-14")
.filters(
new AndDimFilter(
Arrays.asList(
new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "spot", null),
new BoundDimFilter("expr", "11.1", null, false, false, null, null, StringComparators.NUMERIC)
)
)
)
.granularity(QueryRunnerTestHelper.allGran)
.dimensionSpecs(DefaultDimensionSpec.toSpec(QueryRunnerTestHelper.qualityDimension))
.metrics(Lists.<String>newArrayList(QueryRunnerTestHelper.indexMetric))
.pagingSpec(new PagingSpec(null, 10, true))
.virtualColumns(new ExpressionVirtualColumn("expr", "index / 10.0"))
.build();
HashMap<String, Object> context = new HashMap<String, Object>();
Iterable<Result<SelectResultValue>> results = Sequences.toList(
runner.run(query, context),
Lists.<Result<SelectResultValue>>newArrayList()
);
final List<List<Map<String, Object>>> events = toEvents(
new String[]{
EventHolder.timestampKey + ":TIME",
null,
QueryRunnerTestHelper.qualityDimension + ":STRING",
null,
null,
QueryRunnerTestHelper.indexMetric + ":FLOAT"
},
// filtered values with all granularity
new String[]{
"2011-01-13T00:00:00.000Z spot health preferred hpreferred 114.947403",
"2011-01-13T00:00:00.000Z spot technology preferred tpreferred 111.356672"
}
);
PagingOffset offset = query.getPagingOffset(QueryRunnerTestHelper.segmentId);
List<Result<SelectResultValue>> expectedResults = toExpected(
events,
Lists.newArrayList("quality"),
Lists.<String>newArrayList("index"),
offset.startOffset(),
offset.threshold()
);
verify(expectedResults, results);
}
@Test
public void testSelectWithFilterLookupExtractionFn () {

16
processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java
View File

@ -54,6 +54,7 @@ import io.druid.segment.QueryableIndex;
import io.druid.segment.QueryableIndexStorageAdapter;
import io.druid.segment.StorageAdapter;
import io.druid.segment.TestHelper;
import io.druid.segment.VirtualColumn;
import io.druid.segment.VirtualColumns;
import io.druid.segment.column.ValueType;
import io.druid.segment.data.BitmapSerdeFactory;
@ -62,6 +63,7 @@ import io.druid.segment.data.IndexedInts;
import io.druid.segment.data.RoaringBitmapSerdeFactory;
import io.druid.segment.incremental.IncrementalIndex;
import io.druid.segment.incremental.IncrementalIndexStorageAdapter;
import io.druid.segment.virtual.ExpressionVirtualColumn;
import org.joda.time.Interval;
import org.junit.Assert;
import org.junit.Before;
@ -79,6 +81,12 @@ import java.util.Map;
public abstract class BaseFilterTest
{
private static final VirtualColumns VIRTUAL_COLUMNS = VirtualColumns.create(
ImmutableList.<VirtualColumn>of(
new ExpressionVirtualColumn("expr", "1.0 + 0.1")
)
);
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
@ -291,15 +299,13 @@ public abstract class BaseFilterTest
private Sequence<Cursor> makeCursorSequence(final Filter filter)
{
final Sequence<Cursor> cursors = adapter.makeCursors(
return adapter.makeCursors(
filter,
new Interval(JodaUtils.MIN_INSTANT, JodaUtils.MAX_INSTANT),
VirtualColumns.EMPTY,
VIRTUAL_COLUMNS,
QueryGranularities.ALL,
false
);
return cursors;
}
/**
@ -444,7 +450,7 @@ public abstract class BaseFilterTest
// Perform test
final SettableSupplier<InputRow> rowSupplier = new SettableSupplier<>();
final ValueMatcher matcher = makeFilter(filter).makeMatcher(
RowBasedColumnSelectorFactory.create(rowSupplier, rowSignature)
VIRTUAL_COLUMNS.wrap(RowBasedColumnSelectorFactory.create(rowSupplier, rowSignature))
);
final List<String> values = Lists.newArrayList();
for (InputRow row : rows) {

14
processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java
View File

@ -354,6 +354,20 @@ public class BoundFilterTest extends BaseFilterTest
);
}
@Test
public void testNumericMatchVirtualColumn()
{
assertFilterMatches(
new BoundDimFilter("expr", "1", "2", false, false, false, null, StringComparators.NUMERIC),
ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")
);
assertFilterMatches(
new BoundDimFilter("expr", "2", "3", false, false, false, null, StringComparators.NUMERIC),
ImmutableList.<String>of()
);
}
@Test
public void testNumericMatchExactlySingleValue()
{

7
processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java
View File

@ -144,6 +144,13 @@ public class SelectorFilterTest extends BaseFilterTest
assertFilterMatches(new SelectorDimFilter("dim4", "c", null), ImmutableList.<String>of());
}
@Test
public void testExpressionVirtualColumn()
{
assertFilterMatches(new SelectorDimFilter("expr", "1.1", null), ImmutableList.of("0", "1", "2", "3", "4", "5"));
assertFilterMatches(new SelectorDimFilter("expr", "1.2", null), ImmutableList.<String>of());
}
@Test
public void testSelectorWithLookupExtractionFn()
{