diff --git a/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java b/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java index f574c108bed..235ad1a096d 100644 --- a/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java @@ -25,6 +25,7 @@ import io.druid.query.filter.BitmapIndexSelector; import io.druid.segment.ColumnSelectorBitmapIndexSelector; import io.druid.segment.QueryableIndex; import io.druid.segment.Segment; +import io.druid.segment.VirtualColumns; import io.druid.segment.column.BitmapIndex; import io.druid.segment.column.Column; @@ -54,6 +55,7 @@ public class AutoStrategy extends SearchStrategy if (index != null) { final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector( index.getBitmapFactoryForDimensions(), + VirtualColumns.EMPTY, index ); diff --git a/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java b/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java index e68d032224f..d755a87b6b6 100644 --- a/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java @@ -37,6 +37,7 @@ import io.druid.segment.ColumnSelectorBitmapIndexSelector; import io.druid.segment.QueryableIndex; import io.druid.segment.Segment; import io.druid.segment.StorageAdapter; +import io.druid.segment.VirtualColumns; import io.druid.segment.column.BitmapIndex; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; @@ -80,6 +81,7 @@ public class UseIndexesStrategy extends SearchStrategy if (bitmapSuppDims.size() > 0) { final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector( index.getBitmapFactoryForDimensions(), + VirtualColumns.EMPTY, index ); @@ -153,6 +155,7 @@ public class UseIndexesStrategy extends SearchStrategy } else { final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector( index.getBitmapFactoryForDimensions(), + VirtualColumns.EMPTY, index ); Preconditions.checkArgument(filter.supportsBitmapIndex(selector), "filter[%s] should support bitmap", filter); diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java index 9b9d5c2c9f4..23cfabf143a 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -26,6 +26,7 @@ import io.druid.collections.spatial.ImmutableRTree; import io.druid.query.filter.BitmapIndexSelector; import io.druid.segment.column.BitmapIndex; import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.DictionaryEncodedColumn; import io.druid.segment.column.GenericColumn; import io.druid.segment.column.ValueType; @@ -40,20 +41,28 @@ import java.util.Iterator; public class ColumnSelectorBitmapIndexSelector implements BitmapIndexSelector { private final BitmapFactory bitmapFactory; + private final VirtualColumns virtualColumns; private final ColumnSelector index; public ColumnSelectorBitmapIndexSelector( final BitmapFactory bitmapFactory, + final VirtualColumns virtualColumns, final ColumnSelector index ) { this.bitmapFactory = bitmapFactory; + this.virtualColumns = virtualColumns; this.index = index; } @Override public Indexed getDimensionValues(String dimension) { + if (isFilterableVirtualColumn(dimension)) { + // Virtual columns don't have dictionaries or indexes. + return null; + } + final Column columnDesc = index.getColumn(dimension); if (columnDesc == null || !columnDesc.getCapabilities().isDictionaryEncoded()) { return null; @@ -110,6 +119,11 @@ public class ColumnSelectorBitmapIndexSelector implements BitmapIndexSelector @Override public BitmapIndex getBitmapIndex(String dimension) { + if (isFilterableVirtualColumn(dimension)) { + // Virtual columns don't have dictionaries or indexes. + return null; + } + final Column column = index.getColumn(dimension); if (column == null || !columnSupportsFiltering(column)) { // for missing columns and columns with types that do not support filtering, @@ -172,6 +186,11 @@ public class ColumnSelectorBitmapIndexSelector implements BitmapIndexSelector @Override public ImmutableBitmap getBitmapIndex(String dimension, String value) { + if (isFilterableVirtualColumn(dimension)) { + // Virtual columns don't have dictionaries or indexes. + return null; + } + final Column column = index.getColumn(dimension); if (column == null || !columnSupportsFiltering(column)) { if (Strings.isNullOrEmpty(value)) { @@ -192,6 +211,10 @@ public class ColumnSelectorBitmapIndexSelector implements BitmapIndexSelector @Override public ImmutableRTree getSpatialIndex(String dimension) { + if (isFilterableVirtualColumn(dimension)) { + return new ImmutableRTree(); + } + final Column column = index.getColumn(dimension); if (column == null || !column.getCapabilities().hasSpatialIndexes()) { return new ImmutableRTree(); @@ -200,6 +223,16 @@ public class ColumnSelectorBitmapIndexSelector implements BitmapIndexSelector return column.getSpatialIndex().getRTree(); } + private boolean isFilterableVirtualColumn(final String columnName) + { + final ColumnCapabilities columnCapabilities = virtualColumns.getColumnCapabilities(columnName); + if (columnCapabilities == null) { + return false; + } else { + return Filters.FILTERABLE_TYPES.contains(columnCapabilities.getType()); + } + } + private static boolean columnSupportsFiltering(Column column) { ValueType columnType = column.getCapabilities().getType(); diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index dadf2539b5c..ba10fad7966 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -226,6 +226,7 @@ public class QueryableIndexStorageAdapter implements StorageAdapter final ColumnSelectorBitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector( index.getBitmapFactoryForDimensions(), + virtualColumns, index ); diff --git a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java index 3342fd5a486..76eadc50dbb 100644 --- a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java @@ -42,13 +42,16 @@ import io.druid.query.extraction.ExtractionFn; import io.druid.query.extraction.JavaScriptExtractionFn; import io.druid.query.extraction.MapLookupExtractor; import io.druid.query.filter.AndDimFilter; +import io.druid.query.filter.BoundDimFilter; import io.druid.query.filter.DimFilter; import io.druid.query.filter.SelectorDimFilter; import io.druid.query.lookup.LookupExtractionFn; +import io.druid.query.ordering.StringComparators; import io.druid.query.spec.LegacySegmentSpec; import io.druid.query.spec.QuerySegmentSpec; import io.druid.segment.column.Column; import io.druid.segment.column.ValueType; +import io.druid.segment.virtual.ExpressionVirtualColumn; import org.joda.time.DateTime; import org.joda.time.Interval; import org.junit.Assert; @@ -479,6 +482,59 @@ public class SelectQueryRunnerTest } } + @Test + public void testFullOnSelectWithFilterOnVirtualColumn() + { + SelectQuery query = newTestQuery() + .intervals("2011-01-13/2011-01-14") + .filters( + new AndDimFilter( + Arrays.asList( + new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "spot", null), + new BoundDimFilter("expr", "11.1", null, false, false, null, null, StringComparators.NUMERIC) + ) + ) + ) + .granularity(QueryRunnerTestHelper.allGran) + .dimensionSpecs(DefaultDimensionSpec.toSpec(QueryRunnerTestHelper.qualityDimension)) + .metrics(Lists.newArrayList(QueryRunnerTestHelper.indexMetric)) + .pagingSpec(new PagingSpec(null, 10, true)) + .virtualColumns(new ExpressionVirtualColumn("expr", "index / 10.0")) + .build(); + + HashMap context = new HashMap(); + Iterable> results = Sequences.toList( + runner.run(query, context), + Lists.>newArrayList() + ); + + final List>> events = toEvents( + new String[]{ + EventHolder.timestampKey + ":TIME", + null, + QueryRunnerTestHelper.qualityDimension + ":STRING", + null, + null, + QueryRunnerTestHelper.indexMetric + ":FLOAT" + }, + // filtered values with all granularity + new String[]{ + "2011-01-13T00:00:00.000Z spot health preferred hpreferred 114.947403", + "2011-01-13T00:00:00.000Z spot technology preferred tpreferred 111.356672" + } + ); + + PagingOffset offset = query.getPagingOffset(QueryRunnerTestHelper.segmentId); + List> expectedResults = toExpected( + events, + Lists.newArrayList("quality"), + Lists.newArrayList("index"), + offset.startOffset(), + offset.threshold() + ); + verify(expectedResults, results); + } + @Test public void testSelectWithFilterLookupExtractionFn () { diff --git a/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java b/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java index 0c5cc996d1c..63ff837c20e 100644 --- a/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java @@ -54,6 +54,7 @@ import io.druid.segment.QueryableIndex; import io.druid.segment.QueryableIndexStorageAdapter; import io.druid.segment.StorageAdapter; import io.druid.segment.TestHelper; +import io.druid.segment.VirtualColumn; import io.druid.segment.VirtualColumns; import io.druid.segment.column.ValueType; import io.druid.segment.data.BitmapSerdeFactory; @@ -62,6 +63,7 @@ import io.druid.segment.data.IndexedInts; import io.druid.segment.data.RoaringBitmapSerdeFactory; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexStorageAdapter; +import io.druid.segment.virtual.ExpressionVirtualColumn; import org.joda.time.Interval; import org.junit.Assert; import org.junit.Before; @@ -79,6 +81,12 @@ import java.util.Map; public abstract class BaseFilterTest { + private static final VirtualColumns VIRTUAL_COLUMNS = VirtualColumns.create( + ImmutableList.of( + new ExpressionVirtualColumn("expr", "1.0 + 0.1") + ) + ); + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -291,15 +299,13 @@ public abstract class BaseFilterTest private Sequence makeCursorSequence(final Filter filter) { - final Sequence cursors = adapter.makeCursors( + return adapter.makeCursors( filter, new Interval(JodaUtils.MIN_INSTANT, JodaUtils.MAX_INSTANT), - VirtualColumns.EMPTY, + VIRTUAL_COLUMNS, QueryGranularities.ALL, false ); - - return cursors; } /** @@ -444,7 +450,7 @@ public abstract class BaseFilterTest // Perform test final SettableSupplier rowSupplier = new SettableSupplier<>(); final ValueMatcher matcher = makeFilter(filter).makeMatcher( - RowBasedColumnSelectorFactory.create(rowSupplier, rowSignature) + VIRTUAL_COLUMNS.wrap(RowBasedColumnSelectorFactory.create(rowSupplier, rowSignature)) ); final List values = Lists.newArrayList(); for (InputRow row : rows) { diff --git a/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java b/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java index 99969f5d15c..0e11e4ed531 100644 --- a/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java @@ -354,6 +354,20 @@ public class BoundFilterTest extends BaseFilterTest ); } + @Test + public void testNumericMatchVirtualColumn() + { + assertFilterMatches( + new BoundDimFilter("expr", "1", "2", false, false, false, null, StringComparators.NUMERIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + + assertFilterMatches( + new BoundDimFilter("expr", "2", "3", false, false, false, null, StringComparators.NUMERIC), + ImmutableList.of() + ); + } + @Test public void testNumericMatchExactlySingleValue() { diff --git a/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java index b63e79b3678..5eae5e1bdd0 100644 --- a/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java @@ -144,6 +144,13 @@ public class SelectorFilterTest extends BaseFilterTest assertFilterMatches(new SelectorDimFilter("dim4", "c", null), ImmutableList.of()); } + @Test + public void testExpressionVirtualColumn() + { + assertFilterMatches(new SelectorDimFilter("expr", "1.1", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + assertFilterMatches(new SelectorDimFilter("expr", "1.2", null), ImmutableList.of()); + } + @Test public void testSelectorWithLookupExtractionFn() {