Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

remake column indexes and query processing of filters (#12388) 

Following up on #12315, which pushed most of the logic of building ImmutableBitmap into BitmapIndex in order to hide the details of how column indexes are implemented from the Filter implementations, this PR totally refashions how Filter consume indexes. The end result, while a rather dramatic reshuffling of the existing code, should be extraordinarily flexible, eventually allowing us to model any type of index we can imagine, and providing the machinery to build the filters that use them, while also allowing for other column implementations to implement the built-in index types to provide adapters to make use indexing in the current set filters that Druid provides.
This commit is contained in:
Clint Wylie 2022-05-10 23:27:08 -07:00 committed by GitHub
parent deb69d1bc0
commit 9e5a940cf1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
75 changed files with 2854 additions and 2863 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
benchmarks/src/test/java/org/apache/druid/benchmark/BoundFilterBenchmark.java
View File

@ -19,7 +19,6 @@
package org.apache.druid.benchmark;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.FluentIterable;
import org.apache.druid.collections.bitmap.BitmapFactory;
@ -28,15 +27,15 @@ import org.apache.druid.collections.bitmap.MutableBitmap;
import org.apache.druid.collections.bitmap.RoaringBitmapFactory;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.extendedset.intset.ConciseSetUtils;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.BoundDimFilter;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.ordering.StringComparators;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.data.BitmapSerdeFactory;
import org.apache.druid.segment.data.GenericIndexed;
import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
import org.apache.druid.segment.filter.BoundFilter;
import org.apache.druid.segment.serde.StringBitmapIndexColumnPartSupplier;
import org.apache.druid.segment.filter.Filters;
import org.apache.druid.segment.serde.DictionaryEncodedStringIndexSupplier;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
@ -151,7 +150,7 @@ public class BoundFilterBenchmark
int step;
// selector will contain a cardinality number of bitmaps; each one contains a single int: 0
BitmapIndexSelector selector;
ColumnIndexSelector selector;
@Setup
public void setup()
@ -164,27 +163,21 @@ public class BoundFilterBenchmark
FluentIterable.from(ints).transform(i -> i.toString()),
GenericIndexed.STRING_STRATEGY
);
final BitmapIndex bitmapIndex = new StringBitmapIndexColumnPartSupplier(
final GenericIndexed<ImmutableBitmap> bitmaps = GenericIndexed.fromIterable(
FluentIterable.from(ints)
.transform(
i -> {
final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
mutableBitmap.add((i - START_INT) / step);
return bitmapFactory.makeImmutableBitmap(mutableBitmap);
}
),
serdeFactory.getObjectStrategy()
);
selector = new MockColumnIndexSelector(
bitmapFactory,
GenericIndexed.fromIterable(
FluentIterable.from(ints)
.transform(
new Function<Integer, ImmutableBitmap>()
{
@Override
public ImmutableBitmap apply(Integer i)
{
final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
mutableBitmap.add((i - START_INT) / step);
return bitmapFactory.makeImmutableBitmap(mutableBitmap);
}
}
),
serdeFactory.getObjectStrategy()
),
dictionary
).get();
selector = new MockBitmapIndexSelector(dictionary, bitmapFactory, bitmapIndex);
new DictionaryEncodedStringIndexSupplier(bitmapFactory, dictionary, bitmaps, null)
);
}
@Benchmark
@ -192,7 +185,7 @@ public class BoundFilterBenchmark
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void matchNothingLexicographic()
{
final ImmutableBitmap bitmapIndex = NOTHING_LEXICOGRAPHIC.getBitmapIndex(selector);
final ImmutableBitmap bitmapIndex = Filters.computeDefaultBitmapResults(NOTHING_LEXICOGRAPHIC, selector);
Preconditions.checkState(bitmapIndex.size() == 0);
}
@ -201,7 +194,7 @@ public class BoundFilterBenchmark
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void matchHalfLexicographic()
{
final ImmutableBitmap bitmapIndex = HALF_LEXICOGRAPHIC.getBitmapIndex(selector);
final ImmutableBitmap bitmapIndex = Filters.computeDefaultBitmapResults(HALF_LEXICOGRAPHIC, selector);
Preconditions.checkState(bitmapIndex.size() > 0 && bitmapIndex.size() < cardinality);
}
@ -210,7 +203,7 @@ public class BoundFilterBenchmark
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void matchEverythingLexicographic()
{
final ImmutableBitmap bitmapIndex = EVERYTHING_LEXICOGRAPHIC.getBitmapIndex(selector);
final ImmutableBitmap bitmapIndex = Filters.computeDefaultBitmapResults(EVERYTHING_LEXICOGRAPHIC, selector);
Preconditions.checkState(bitmapIndex.size() == cardinality);
}
@ -219,7 +212,7 @@ public class BoundFilterBenchmark
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void matchNothingAlphaNumeric()
{
final ImmutableBitmap bitmapIndex = NOTHING_ALPHANUMERIC.getBitmapIndex(selector);
final ImmutableBitmap bitmapIndex = Filters.computeDefaultBitmapResults(NOTHING_ALPHANUMERIC, selector);
Preconditions.checkState(bitmapIndex.size() == 0);
}
@ -228,7 +221,7 @@ public class BoundFilterBenchmark
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void matchHalfAlphaNumeric()
{
final ImmutableBitmap bitmapIndex = HALF_ALPHANUMERIC.getBitmapIndex(selector);
final ImmutableBitmap bitmapIndex = Filters.computeDefaultBitmapResults(HALF_ALPHANUMERIC, selector);
Preconditions.checkState(bitmapIndex.size() > 0 && bitmapIndex.size() < cardinality);
}
@ -237,7 +230,7 @@ public class BoundFilterBenchmark
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void matchEverythingAlphaNumeric()
{
final ImmutableBitmap bitmapIndex = EVERYTHING_ALPHANUMERIC.getBitmapIndex(selector);
final ImmutableBitmap bitmapIndex = Filters.computeDefaultBitmapResults(EVERYTHING_ALPHANUMERIC, selector);
Preconditions.checkState(bitmapIndex.size() == cardinality);
}

44
benchmarks/src/test/java/org/apache/druid/benchmark/DimensionPredicateFilterBenchmark.java
View File

@ -28,17 +28,17 @@ import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.collections.bitmap.MutableBitmap;
import org.apache.druid.collections.bitmap.RoaringBitmapFactory;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.DruidDoublePredicate;
import org.apache.druid.query.filter.DruidFloatPredicate;
import org.apache.druid.query.filter.DruidLongPredicate;
import org.apache.druid.query.filter.DruidPredicateFactory;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.data.BitmapSerdeFactory;
import org.apache.druid.segment.data.GenericIndexed;
import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
import org.apache.druid.segment.filter.DimensionPredicateFilter;
import org.apache.druid.segment.serde.StringBitmapIndexColumnPartSupplier;
import org.apache.druid.segment.filter.Filters;
import org.apache.druid.segment.serde.DictionaryEncodedStringIndexSupplier;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
@ -113,7 +113,7 @@ public class DimensionPredicateFilterBenchmark
int cardinality;
// selector will contain a cardinality number of bitmaps; each one contains a single int: 0
BitmapIndexSelector selector;
ColumnIndexSelector selector;
@Setup
public void setup()
@ -135,27 +135,21 @@ public class DimensionPredicateFilterBenchmark
),
GenericIndexed.STRING_STRATEGY
);
final BitmapIndex bitmapIndex = new StringBitmapIndexColumnPartSupplier(
final GenericIndexed<ImmutableBitmap> bitmaps = GenericIndexed.fromIterable(
FluentIterable.from(ints)
.transform(
i -> {
final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
mutableBitmap.add(i - START_INT);
return bitmapFactory.makeImmutableBitmap(mutableBitmap);
}
),
serdeFactory.getObjectStrategy()
);
selector = new MockColumnIndexSelector(
bitmapFactory,
GenericIndexed.fromIterable(
FluentIterable.from(ints)
.transform(
new Function<Integer, ImmutableBitmap>()
{
@Override
public ImmutableBitmap apply(Integer i)
{
final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
mutableBitmap.add(i - START_INT);
return bitmapFactory.makeImmutableBitmap(mutableBitmap);
}
}
),
serdeFactory.getObjectStrategy()
),
dictionary
).get();
selector = new MockBitmapIndexSelector(dictionary, bitmapFactory, bitmapIndex);
new DictionaryEncodedStringIndexSupplier(bitmapFactory, dictionary, bitmaps, null)
);
}
@Benchmark
@ -163,7 +157,7 @@ public class DimensionPredicateFilterBenchmark
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void matchIsEven()
{
final ImmutableBitmap bitmapIndex = IS_EVEN.getBitmapIndex(selector);
final ImmutableBitmap bitmapIndex = Filters.computeDefaultBitmapResults(IS_EVEN, selector);
Preconditions.checkState(bitmapIndex.size() == cardinality / 2);
}

11
benchmarks/src/test/java/org/apache/druid/benchmark/FilterPartitionBenchmark.java
View File

@ -36,7 +36,6 @@ import org.apache.druid.query.dimension.DefaultDimensionSpec;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.query.extraction.JavaScriptExtractionFn;
import org.apache.druid.query.filter.AndDimFilter;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.BoundDimFilter;
import org.apache.druid.query.filter.DimFilter;
import org.apache.druid.query.filter.DruidDoublePredicate;
@ -513,11 +512,6 @@ public class FilterPartitionBenchmark
super(dimension, value);
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return false;
}
}
private static class NoBitmapDimensionPredicateFilter extends DimensionPredicateFilter
@ -531,11 +525,6 @@ public class FilterPartitionBenchmark
super(dimension, predicateFactory, extractionFn);
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return false;
}
}
private static class NoBitmapSelectorDimFilter extends SelectorDimFilter

62
benchmarks/src/test/java/org/apache/druid/benchmark/LikeFilterBenchmark.java
View File

@ -19,25 +19,24 @@
package org.apache.druid.benchmark;
import com.google.common.base.Function;
import com.google.common.collect.FluentIterable;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.collections.bitmap.MutableBitmap;
import org.apache.druid.collections.bitmap.RoaringBitmapFactory;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.BoundDimFilter;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.LikeDimFilter;
import org.apache.druid.query.filter.RegexDimFilter;
import org.apache.druid.query.filter.SelectorDimFilter;
import org.apache.druid.query.ordering.StringComparators;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.data.BitmapSerdeFactory;
import org.apache.druid.segment.data.GenericIndexed;
import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
import org.apache.druid.segment.serde.StringBitmapIndexColumnPartSupplier;
import org.apache.druid.segment.filter.Filters;
import org.apache.druid.segment.serde.DictionaryEncodedStringIndexSupplier;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
@ -112,7 +111,7 @@ public class LikeFilterBenchmark
int step;
// selector will contain a cardinality number of bitmaps; each one contains a single int: 0
BitmapIndexSelector selector;
ColumnIndexSelector selector;
@Setup
public void setup()
@ -124,38 +123,25 @@ public class LikeFilterBenchmark
final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(
FluentIterable.from(ints)
.transform(
new Function<Integer, String>()
{
@Override
public String apply(Integer i)
{
return i.toString();
}
}
i -> i.toString()
),
GenericIndexed.STRING_STRATEGY
);
final BitmapIndex bitmapIndex = new StringBitmapIndexColumnPartSupplier(
final GenericIndexed<ImmutableBitmap> bitmaps = GenericIndexed.fromIterable(
FluentIterable.from(ints)
.transform(
i -> {
final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
mutableBitmap.add((i - START_INT) / step);
return bitmapFactory.makeImmutableBitmap(mutableBitmap);
}
),
serdeFactory.getObjectStrategy()
);
selector = new MockColumnIndexSelector(
bitmapFactory,
GenericIndexed.fromIterable(
FluentIterable.from(ints)
.transform(
new Function<Integer, ImmutableBitmap>()
{
@Override
public ImmutableBitmap apply(Integer i)
{
final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
mutableBitmap.add((i - START_INT) / step);
return bitmapFactory.makeImmutableBitmap(mutableBitmap);
}
}
),
serdeFactory.getObjectStrategy()
),
dictionary
).get();
selector = new MockBitmapIndexSelector(dictionary, bitmapFactory, bitmapIndex);
new DictionaryEncodedStringIndexSupplier(bitmapFactory, dictionary, bitmaps, null)
);
}
@Benchmark
@ -163,7 +149,7 @@ public class LikeFilterBenchmark
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void matchLikeEquals(Blackhole blackhole)
{
final ImmutableBitmap bitmapIndex = LIKE_EQUALS.getBitmapIndex(selector);
final ImmutableBitmap bitmapIndex = Filters.computeDefaultBitmapResults(LIKE_EQUALS, selector);
blackhole.consume(bitmapIndex);
}
@ -172,7 +158,7 @@ public class LikeFilterBenchmark
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void matchSelectorEquals(Blackhole blackhole)
{
final ImmutableBitmap bitmapIndex = SELECTOR_EQUALS.getBitmapIndex(selector);
final ImmutableBitmap bitmapIndex = Filters.computeDefaultBitmapResults(SELECTOR_EQUALS, selector);
blackhole.consume(bitmapIndex);
}
@ -181,7 +167,7 @@ public class LikeFilterBenchmark
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void matchLikePrefix(Blackhole blackhole)
{
final ImmutableBitmap bitmapIndex = LIKE_PREFIX.getBitmapIndex(selector);
final ImmutableBitmap bitmapIndex = Filters.computeDefaultBitmapResults(LIKE_PREFIX, selector);
blackhole.consume(bitmapIndex);
}
@ -190,7 +176,7 @@ public class LikeFilterBenchmark
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void matchBoundPrefix(Blackhole blackhole)
{
final ImmutableBitmap bitmapIndex = BOUND_PREFIX.getBitmapIndex(selector);
final ImmutableBitmap bitmapIndex = Filters.computeDefaultBitmapResults(BOUND_PREFIX, selector);
blackhole.consume(bitmapIndex);
}
@ -199,7 +185,7 @@ public class LikeFilterBenchmark
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void matchRegexPrefix(Blackhole blackhole)
{
final ImmutableBitmap bitmapIndex = REGEX_PREFIX.getBitmapIndex(selector);
final ImmutableBitmap bitmapIndex = Filters.computeDefaultBitmapResults(REGEX_PREFIX, selector);
blackhole.consume(bitmapIndex);
}

51
benchmarks/src/test/java/org/apache/druid/benchmark/MockBitmapIndexSelector.java → benchmarks/src/test/java/org/apache/druid/benchmark/MockColumnIndexSelector.java
View File

@ -20,42 +20,24 @@
package org.apache.druid.benchmark;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.collections.spatial.ImmutableRTree;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.data.CloseableIndexed;
import org.apache.druid.segment.data.GenericIndexed;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import javax.annotation.Nullable;
public class MockBitmapIndexSelector implements BitmapIndexSelector
public class MockColumnIndexSelector implements ColumnIndexSelector
{
private final GenericIndexed<String> dictionary;
private final BitmapFactory bitmapFactory;
private final BitmapIndex bitmapIndex;
private final ColumnIndexSupplier indexSupplier;
public MockBitmapIndexSelector(
GenericIndexed<String> dictionary,
public MockColumnIndexSelector(
BitmapFactory bitmapFactory,
BitmapIndex bitmapIndex)
ColumnIndexSupplier indexSupplier
)
{
this.dictionary = dictionary;
this.bitmapFactory = bitmapFactory;
this.bitmapIndex = bitmapIndex;
}
@Override
public CloseableIndexed<String> getDimensionValues(String dimension)
{
return dictionary;
}
@Override
public ColumnCapabilities.Capable hasMultipleValues(final String dimension)
{
throw new UnsupportedOperationException();
this.indexSupplier = indexSupplier;
}
@Override
@ -71,21 +53,9 @@ public class MockBitmapIndexSelector implements BitmapIndexSelector
}
@Override
public ImmutableBitmap getBitmapIndex(String dimension, String value)
public ColumnIndexSupplier getIndexSupplier(String column)
{
return bitmapIndex.getBitmapForValue(value);
}
@Override
public BitmapIndex getBitmapIndex(String dimension)
{
return bitmapIndex;
}
@Override
public ImmutableRTree getSpatialIndex(String dimension)
{
throw new UnsupportedOperationException();
return indexSupplier;
}
@Nullable
@ -94,4 +64,5 @@ public class MockBitmapIndexSelector implements BitmapIndexSelector
{
return null;
}
}

21
indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java
View File

@ -101,12 +101,11 @@ import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.SegmentUtils;
import org.apache.druid.segment.SimpleQueryableIndex;
import org.apache.druid.segment.column.BaseColumn;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.SpatialIndex;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.data.CompressionFactory.LongEncodingStrategy;
import org.apache.druid.segment.data.CompressionStrategy;
@ -2178,23 +2177,19 @@ public class CompactionTaskTest
return null;
}
@Nullable
@Override
public ColumnIndexSupplier getIndexSupplier()
{
return null;
}
@Override
public SettableColumnValueSelector<?> makeNewSettableColumnValueSelector()
{
return null;
}
@Override
public BitmapIndex getBitmapIndex()
{
return null;
}
@Override
public SpatialIndex getSpatialIndex()
{
return null;
}
}
/**

28
processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java
View File

@ -45,7 +45,7 @@ public interface BooleanFilter extends Filter
*
* An implementation should either:
* - return a ValueMatcher that checks row values, using the provided ValueMatcherFactory
* - or, if possible, get a bitmap index for this filter using the BitmapIndexSelector, and
* - or, if possible, get a bitmap index for this filter using the ColumnIndexSelector, and
* return a ValueMatcher that checks the current row offset, created using the bitmap index.
*
* @param selector Object used to retrieve bitmap indexes
@ -55,7 +55,7 @@ public interface BooleanFilter extends Filter
* @return ValueMatcher that applies this filter
*/
ValueMatcher makeMatcher(
BitmapIndexSelector selector,
ColumnIndexSelector selector,
ColumnSelectorFactory columnSelectorFactory,
RowOffsetMatcherFactory rowOffsetMatcherFactory
);
@ -71,29 +71,7 @@ public interface BooleanFilter extends Filter
}
@Override
default boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
for (Filter filter : getFilters()) {
if (!filter.supportsBitmapIndex(selector)) {
return false;
}
}
return true;
}
@Override
default boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
for (Filter f : getFilters()) {
if (!f.shouldUseBitmapIndex(selector)) {
return false;
}
}
return true;
}
@Override
default boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
default boolean supportsSelectivityEstimation(ColumnSelector columnSelector, ColumnIndexSelector indexSelector)
{
for (Filter filter : getFilters()) {
if (!filter.supportsSelectivityEstimation(columnSelector, indexSelector)) {

29
processing/src/main/java/org/apache/druid/query/filter/BitmapIndexSelector.java → processing/src/main/java/org/apache/druid/query/filter/ColumnIndexSelector.java
View File

@ -19,33 +19,26 @@
package org.apache.druid.query.filter;
import com.google.errorprone.annotations.MustBeClosed;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.collections.spatial.ImmutableRTree;
import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.data.CloseableIndexed;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import javax.annotation.Nullable;
/**
*/
public interface BitmapIndexSelector extends ColumnInspector
public interface ColumnIndexSelector extends ColumnInspector
{
@MustBeClosed
@Nullable
CloseableIndexed<String> getDimensionValues(String dimension);
@Deprecated
ColumnCapabilities.Capable hasMultipleValues(String dimension);
int getNumRows();
BitmapFactory getBitmapFactory();
/**
* Get the {@link ColumnIndexSupplier} of a column. If the column exists, but does not support indexes, this method
* will return a non-null index supplier, likely {@link org.apache.druid.segment.serde.NoIndexesColumnIndexSupplier}.
* Columns which are 'missing' will return a null value from this method, which allows for filters to act on this
* information to produce an all true or all false index depending on how the filter matches the null value.
*/
@Nullable
BitmapIndex getBitmapIndex(String dimension);
@Nullable
ImmutableBitmap getBitmapIndex(String dimension, String value);
ImmutableRTree getSpatialIndex(String dimension);
ColumnIndexSupplier getIndexSupplier(String column);
}

102
processing/src/main/java/org/apache/druid/query/filter/Filter.java
View File

@ -20,16 +20,16 @@
package org.apache.druid.query.filter;
import org.apache.druid.annotations.SubclassesMustOverrideEqualsAndHashCode;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.java.util.common.UOE;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.DefaultBitmapResultFactory;
import org.apache.druid.query.filter.vector.VectorValueMatcher;
import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import javax.annotation.Nullable;
import java.util.Map;
import java.util.Set;
@ -37,51 +37,17 @@ import java.util.Set;
public interface Filter
{
/**
* Get a bitmap index, indicating rows that match this filter. Do not call this method unless
* {@link #supportsBitmapIndex(BitmapIndexSelector)} returns true. Behavior in the case that
* {@link #supportsBitmapIndex(BitmapIndexSelector)} returns false is undefined.
* Returns a {@link BitmapColumnIndex} if this filter supports using a bitmap index for filtering for the given input
* {@link ColumnIndexSelector}. The {@link BitmapColumnIndex} can be used to compute into a bitmap indicating rows
* that match this filter result {@link BitmapColumnIndex#computeBitmapResult(BitmapResultFactory)}, or examine
* details about the index prior to computing it, via {@link BitmapColumnIndex#getIndexCapabilities()}.
*
* This method is OK to be called, but generally should not be overridden, override {@link #getBitmapResult} instead.
* @param selector Object used to create BitmapColumnIndex
*
* @param selector Object used to retrieve bitmap indexes
*
* @return A bitmap indicating rows that match this filter.
*
* @see Filter#estimateSelectivity(BitmapIndexSelector)
* @return BitmapColumnIndex that can build ImmutableBitmap of matched row numbers
*/
default ImmutableBitmap getBitmapIndex(BitmapIndexSelector selector)
{
return getBitmapResult(selector, new DefaultBitmapResultFactory(selector.getBitmapFactory()));
}
/**
* Get a (possibly wrapped) bitmap index, indicating rows that match this filter. Do not call this method unless
* {@link #supportsBitmapIndex(BitmapIndexSelector)} returns true. Behavior in the case that
* {@link #supportsBitmapIndex(BitmapIndexSelector)} returns false is undefined.
*
* @param selector Object used to retrieve bitmap indexes
*
* @return A bitmap indicating rows that match this filter.
*
* @see Filter#estimateSelectivity(BitmapIndexSelector)
*/
<T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory);
/**
* Estimate selectivity of this filter.
* This method can be used for cost-based query planning like in {@link org.apache.druid.query.search.AutoStrategy}.
* To avoid significant performance degradation for calculating the exact cost,
* implementation of this method targets to achieve rapid selectivity estimation
* with reasonable sacrifice of the accuracy.
* As a result, the estimated selectivity might be different from the exact value.
*
* @param indexSelector Object used to retrieve bitmap indexes
*
* @return an estimated selectivity ranging from 0 (filter selects no rows) to 1 (filter selects all rows).
*
* @see Filter#getBitmapIndex(BitmapIndexSelector)
*/
double estimateSelectivity(BitmapIndexSelector indexSelector);
@Nullable
BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector);
/**
* Get a ValueMatcher that applies this filter to row values.
@ -105,40 +71,23 @@ public interface Filter
}
/**
* Indicates whether this filter can return a bitmap index for filtering, based on the information provided by the
* input {@link BitmapIndexSelector}.
* Estimate selectivity of this filter.
* This method can be used for cost-based query planning like in {@link org.apache.druid.query.search.AutoStrategy}.
* To avoid significant performance degradation for calculating the exact cost,
* implementation of this method targets to achieve rapid selectivity estimation
* with reasonable sacrifice of the accuracy.
* As a result, the estimated selectivity might be different from the exact value.
*
* Returning a value of true here guarantees that {@link #getBitmapIndex(BitmapIndexSelector)} will return a non-null
* {@link BitmapIndexSelector}, and also that all columns specified in {@link #getRequiredColumns()} have a bitmap
* index retrievable via {@link BitmapIndexSelector#getBitmapIndex(String)}.
* @param indexSelector Object used to retrieve indexes
*
* @param selector Object used to retrieve bitmap indexes
* @return an estimated selectivity ranging from 0 (filter selects no rows) to 1 (filter selects all rows).
*
* @return true if this Filter can provide a bitmap index using the selector, false otherwise.
* @see Filter#getBitmapColumnIndex(ColumnIndexSelector)
*/
boolean supportsBitmapIndex(BitmapIndexSelector selector);
/**
* Determine if a filter *should* use a bitmap index based on information collected from the supplied
* {@link BitmapIndexSelector}. This method differs from {@link #supportsBitmapIndex(BitmapIndexSelector)} in that
* the former only indicates if a bitmap index is available and {@link #getBitmapIndex(BitmapIndexSelector)} may be
* used.
*
* If shouldUseFilter(selector) returns true, {@link #supportsBitmapIndex} must also return true when called with the
* same selector object. Returning a value of true here guarantees that {@link #getBitmapIndex(BitmapIndexSelector)}
* will return a non-null {@link BitmapIndexSelector}, and also that all columns specified in
* {@link #getRequiredColumns()} have a bitmap index retrievable via
* {@link BitmapIndexSelector#getBitmapIndex(String)}.
*
* Implementations of this methods typically consider a {@link FilterTuning} to make decisions about when to
* use an available index. A "standard" implementation of this is available to all {@link Filter} implementations in
* {@link org.apache.druid.segment.filter.Filters#shouldUseBitmapIndex(Filter, BitmapIndexSelector, FilterTuning)}.
*
* @param selector Object used to retrieve bitmap indexes and provide information about the column
*
* @return true if this Filter should provide a bitmap index using the selector, false otherwise.
*/
boolean shouldUseBitmapIndex(BitmapIndexSelector selector);
default double estimateSelectivity(ColumnIndexSelector indexSelector)
{
return getBitmapColumnIndex(indexSelector).estimateSelectivity(indexSelector.getNumRows());
}
/**
* Indicates whether this filter supports selectivity estimation.
@ -150,7 +99,7 @@ public interface Filter
*
* @return true if this Filter supports selectivity estimation, false otherwise.
*/
boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector);
boolean supportsSelectivityEstimation(ColumnSelector columnSelector, ColumnIndexSelector indexSelector);
/**
* Returns true if this filter can produce a vectorized matcher from its "makeVectorMatcher" method.
@ -162,8 +111,7 @@ public interface Filter
}
/**
* Set of columns used by a filter. If {@link #supportsBitmapIndex} returns true, all columns returned by this method
* can be expected to have a bitmap index retrievable via {@link BitmapIndexSelector#getBitmapIndex(String)}
* Set of columns used by a filter.
*/
Set<String> getRequiredColumns();

30
processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java
View File

@ -20,6 +20,7 @@
package org.apache.druid.query.filter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.druid.annotations.SubclassesMustOverrideEqualsAndHashCode;
@ -49,14 +50,11 @@ import java.util.Objects;
@SubclassesMustOverrideEqualsAndHashCode
public class FilterTuning
{
public static FilterTuning createDefault(Filter filter, BitmapIndexSelector selector)
{
return new FilterTuning(filter.supportsBitmapIndex(selector), null, null);
}
private final boolean useBitmapIndex;
private final int minCardinalityToUseBitmapIndex;
private final int maxCardinalityToUseBitmapIndex;
@Nullable
private final Integer minCardinalityToUseBitmapIndex;
@Nullable
private final Integer maxCardinalityToUseBitmapIndex;
@JsonCreator
public FilterTuning(
@ -66,10 +64,8 @@ public class FilterTuning
)
{
this.useBitmapIndex = useBitmapIndex != null ? useBitmapIndex : true;
this.minCardinalityToUseBitmapIndex =
minCardinalityToUseBitmapIndex != null ? minCardinalityToUseBitmapIndex : 0;
this.maxCardinalityToUseBitmapIndex =
maxCardinalityToUseBitmapIndex != null ? maxCardinalityToUseBitmapIndex : Integer.MAX_VALUE;
this.minCardinalityToUseBitmapIndex = minCardinalityToUseBitmapIndex;
this.maxCardinalityToUseBitmapIndex = maxCardinalityToUseBitmapIndex;
}
@JsonProperty
@ -78,18 +74,26 @@ public class FilterTuning
return useBitmapIndex;
}
@Nullable
@JsonProperty
public int getMinCardinalityToUseBitmapIndex()
public Integer getMinCardinalityToUseBitmapIndex()
{
return minCardinalityToUseBitmapIndex;
}
@Nullable
@JsonProperty
public int getMaxCardinalityToUseBitmapIndex()
public Integer getMaxCardinalityToUseBitmapIndex()
{
return maxCardinalityToUseBitmapIndex;
}
@JsonIgnore
public boolean hasValueCardinalityThreshold()
{
return minCardinalityToUseBitmapIndex != null || maxCardinalityToUseBitmapIndex != null;
}
@Override
public boolean equals(Object o)
{

74
processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java
View File

@ -41,12 +41,10 @@ import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.guava.Comparators;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.query.filter.vector.VectorValueMatcher;
@ -58,7 +56,9 @@ import org.apache.druid.segment.ColumnProcessors;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.DimensionHandlerUtils;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.StringValueSetIndex;
import org.apache.druid.segment.filter.Filters;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
@ -280,37 +280,32 @@ public class InDimFilter extends AbstractOptimizableDimFilter implements Filter
}
@Override
public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory)
@Nullable
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
if (extractionFn == null) {
final BitmapIndex bitmapIndex = selector.getBitmapIndex(dimension);
return bitmapResultFactory.unionDimensionValueBitmaps(getBitmapIterable(values, bitmapIndex));
} else {
return Filters.matchPredicate(
dimension,
selector,
bitmapResultFactory,
predicateFactory.makeStringPredicate()
);
if (!Filters.checkFilterTuningUseIndex(dimension, selector, filterTuning)) {
return null;
}
}
if (extractionFn == null) {
final ColumnIndexSupplier indexSupplier = selector.getIndexSupplier(dimension);
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
{
if (extractionFn == null) {
final BitmapIndex bitmapIndex = indexSelector.getBitmapIndex(dimension);
return Filters.estimateSelectivity(
bitmapIndex.getBitmapsForValues(values).iterator(),
indexSelector.getNumRows()
);
} else {
return Filters.estimateSelectivity(
dimension,
indexSelector,
predicateFactory.makeStringPredicate()
);
if (indexSupplier == null) {
// column doesn't exist, match against null
return Filters.makeNullIndex(
predicateFactory.makeStringPredicate().apply(null),
selector
);
}
final StringValueSetIndex valueSetIndex = indexSupplier.as(StringValueSetIndex.class);
if (valueSetIndex != null) {
return valueSetIndex.forValues(values);
}
}
return Filters.makePredicateIndex(
dimension,
selector,
predicateFactory
);
}
@Override
@ -363,19 +358,7 @@ public class InDimFilter extends AbstractOptimizableDimFilter implements Filter
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return selector.getBitmapIndex(dimension) != null;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return Filters.shouldUseBitmapIndex(this, selector, filterTuning);
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, ColumnIndexSelector indexSelector)
{
return Filters.supportsSelectivityEstimation(this, dimension, columnSelector, indexSelector);
}
@ -496,11 +479,6 @@ public class InDimFilter extends AbstractOptimizableDimFilter implements Filter
return comparator == null || Comparators.naturalNullsFirst().equals(comparator);
}
private static Iterable<ImmutableBitmap> getBitmapIterable(final Set<String> values, final BitmapIndex bitmapIndex)
{
return bitmapIndex.getBitmapsForValues(values);
}
@SuppressWarnings("ReturnValueIgnored")
private static Predicate<String> createStringPredicate(final Set<String> values)
{

2
processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java
View File

@ -249,7 +249,7 @@ public class JavaScriptDimFilter extends AbstractOptimizableDimFilter implements
@Override
public Predicate<String> makeStringPredicate()
{
return input -> applyObject(input);
return this::applyObject;
}
@Override

22
processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java
View File

@ -37,13 +37,14 @@ import org.apache.druid.segment.Segment;
import org.apache.druid.segment.StorageAdapter;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.BaseColumn;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.ColumnTypeFactory;
import org.apache.druid.segment.column.ComplexColumn;
import org.apache.druid.segment.column.DictionaryEncodedColumn;
import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.column.TypeSignature;
import org.apache.druid.segment.column.ValueType;
@ -200,23 +201,22 @@ public class SegmentAnalyzer
Comparable max = null;
long size = 0;
final int cardinality;
if (capabilities.hasBitmapIndexes()) {
final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
cardinality = bitmapIndex.getCardinality();
final ColumnIndexSupplier indexSupplier = columnHolder.getIndexSupplier();
final DictionaryEncodedStringValueIndex valueIndex =
indexSupplier == null ? null : indexSupplier.as(DictionaryEncodedStringValueIndex.class);
if (valueIndex != null) {
cardinality = valueIndex.getCardinality();
if (analyzingSize()) {
for (int i = 0; i < cardinality; ++i) {
String value = bitmapIndex.getValue(i);
String value = valueIndex.getValue(i);
if (value != null) {
size += StringUtils.estimatedBinaryLengthAsUTF8(value) *
((long) bitmapIndex.getBitmapForValue(value).size());
size += StringUtils.estimatedBinaryLengthAsUTF8(value) * ((long) valueIndex.getBitmap(i).size());
}
}
}
if (analyzingMinMax() && cardinality > 0) {
min = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(0));
max = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(cardinality - 1));
min = NullHandling.nullToEmptyIfNeeded(valueIndex.getValue(0));
max = NullHandling.nullToEmptyIfNeeded(valueIndex.getValue(cardinality - 1));
}
} else if (capabilities.isDictionaryEncoded().isTrue()) {
// fallback if no bitmap index

19
processing/src/main/java/org/apache/druid/query/search/AutoStrategy.java
View File

@ -21,13 +21,14 @@ package org.apache.druid.query.search;
import org.apache.druid.java.util.emitter.EmittingLogger;
import org.apache.druid.query.dimension.DimensionSpec;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.segment.ColumnSelectorBitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.segment.ColumnSelectorColumnIndexSelector;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.Segment;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex;
import java.util.List;
@ -53,7 +54,7 @@ public class AutoStrategy extends SearchStrategy
final QueryableIndex index = segment.asQueryableIndex();
if (index != null) {
final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(
final ColumnIndexSelector selector = new ColumnSelectorColumnIndexSelector(
index.getBitmapFactoryForDimensions(),
VirtualColumns.EMPTY,
index
@ -116,9 +117,13 @@ public class AutoStrategy extends SearchStrategy
for (DimensionSpec dimension : dimensionSpecs) {
final ColumnHolder columnHolder = index.getColumnHolder(dimension.getDimension());
if (columnHolder != null) {
final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
if (bitmapIndex != null) {
totalCard += bitmapIndex.getCardinality();
final ColumnIndexSupplier indexSupplier = columnHolder.getIndexSupplier();
if (indexSupplier != null) {
final DictionaryEncodedStringValueIndex bitmapIndex =
indexSupplier.as(DictionaryEncodedStringValueIndex.class);
if (bitmapIndex != null) {
totalCard += bitmapIndex.getCardinality();
}
}
}
}

55
processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java
View File

@ -26,20 +26,23 @@ import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.collections.bitmap.MutableBitmap;
import org.apache.druid.java.util.common.Pair;
import org.apache.druid.query.DefaultBitmapResultFactory;
import org.apache.druid.query.dimension.DimensionSpec;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.query.extraction.IdentityExtractionFn;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.search.CursorOnlyStrategy.CursorBasedExecutor;
import org.apache.druid.segment.ColumnSelectorBitmapIndexSelector;
import org.apache.druid.segment.ColumnSelectorColumnIndexSelector;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.Segment;
import org.apache.druid.segment.StorageAdapter;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex;
import org.apache.druid.segment.column.NumericColumn;
import org.joda.time.Interval;
@ -78,7 +81,7 @@ public class UseIndexesStrategy extends SearchStrategy
final List<DimensionSpec> nonBitmapSuppDims = pair.rhs;
if (bitmapSuppDims.size() > 0) {
final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(
final ColumnIndexSelector selector = new ColumnSelectorColumnIndexSelector(
index.getBitmapFactoryForDimensions(),
VirtualColumns.EMPTY,
index
@ -89,7 +92,7 @@ public class UseIndexesStrategy extends SearchStrategy
// Note: if some filters support bitmap indexes but others are not, the current implementation always employs
// the cursor-based plan. This can be more optimized. One possible optimization is generating a bitmap index
// from the non-bitmap-support filter, and then use it to compute the filtered result by intersecting bitmaps.
if (filter == null || filter.supportsBitmapIndex(selector)) {
if (filter == null || filter.getBitmapColumnIndex(selector) != null) {
final ImmutableBitmap timeFilteredBitmap = makeTimeFilteredBitmap(index, segment, filter, interval);
builder.add(new IndexOnlyExecutor(query, segment, timeFilteredBitmap, bitmapSuppDims));
} else {
@ -152,13 +155,18 @@ public class UseIndexesStrategy extends SearchStrategy
if (filter == null) {
baseFilter = null;
} else {
final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(
final ColumnIndexSelector selector = new ColumnSelectorColumnIndexSelector(
index.getBitmapFactoryForDimensions(),
VirtualColumns.EMPTY,
index
);
Preconditions.checkArgument(filter.supportsBitmapIndex(selector), "filter[%s] should support bitmap", filter);
baseFilter = filter.getBitmapIndex(selector);
final BitmapColumnIndex columnIndex = filter.getBitmapColumnIndex(selector);
Preconditions.checkNotNull(
columnIndex,
"filter[%s] should support bitmap",
filter
);
baseFilter = columnIndex.computeBitmapResult(new DefaultBitmapResultFactory(selector.getBitmapFactory()));
}
final ImmutableBitmap timeFilteredBitmap;
@ -251,21 +259,19 @@ public class UseIndexesStrategy extends SearchStrategy
continue;
}
final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
Preconditions.checkArgument(bitmapIndex != null,
"Dimension [%s] should support bitmap index", dimension.getDimension()
);
final ColumnIndexSupplier indexSupplier = columnHolder.getIndexSupplier();
ExtractionFn extractionFn = dimension.getExtractionFn();
if (extractionFn == null) {
extractionFn = IdentityExtractionFn.getInstance();
}
for (int i = 0; i < bitmapIndex.getCardinality(); ++i) {
String dimVal = extractionFn.apply(bitmapIndex.getValue(i));
// if indexSupplier is null here, it means the column is missing
if (indexSupplier == null) {
String dimVal = extractionFn.apply(null);
if (!searchQuerySpec.accept(dimVal)) {
continue;
}
ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
ImmutableBitmap bitmap = bitmapFactory.complement(bitmapFactory.makeEmptyImmutableBitmap(), index.getNumRows());
if (timeFilteredBitmap != null) {
bitmap = bitmapFactory.intersection(Arrays.asList(timeFilteredBitmap, bitmap));
}
@ -275,6 +281,25 @@ public class UseIndexesStrategy extends SearchStrategy
return retVal;
}
}
} else {
final DictionaryEncodedStringValueIndex bitmapIndex =
indexSupplier.as(DictionaryEncodedStringValueIndex.class);
for (int i = 0; i < bitmapIndex.getCardinality(); ++i) {
String dimVal = extractionFn.apply(bitmapIndex.getValue(i));
if (!searchQuerySpec.accept(dimVal)) {
continue;
}
ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
if (timeFilteredBitmap != null) {
bitmap = bitmapFactory.intersection(Arrays.asList(timeFilteredBitmap, bitmap));
}
if (!bitmap.isEmpty()) {
retVal.addTo(new SearchHit(dimension.getOutputName(), dimVal), bitmap.size());
if (retVal.size() >= limit) {
return retVal;
}
}
}
}
}

282
processing/src/main/java/org/apache/druid/segment/ColumnSelectorBitmapIndexSelector.java
View File

@ -1,282 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.collections.spatial.ImmutableRTree;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.column.BaseColumn;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.BitmapIndexes;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.DictionaryEncodedColumn;
import org.apache.druid.segment.column.NumericColumn;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.data.CloseableIndexed;
import org.apache.druid.segment.data.IndexedIterable;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.Iterator;
/**
*/
public class ColumnSelectorBitmapIndexSelector implements BitmapIndexSelector
{
private final BitmapFactory bitmapFactory;
private final VirtualColumns virtualColumns;
private final ColumnSelector index;
public ColumnSelectorBitmapIndexSelector(
final BitmapFactory bitmapFactory,
final VirtualColumns virtualColumns,
final ColumnSelector index
)
{
this.bitmapFactory = bitmapFactory;
this.virtualColumns = virtualColumns;
this.index = index;
}
@Nullable
@Override
public CloseableIndexed<String> getDimensionValues(String dimension)
{
if (isVirtualColumn(dimension)) {
BitmapIndex bitmapIndex = virtualColumns.getBitmapIndex(dimension, index);
if (bitmapIndex == null) {
return null;
}
return new CloseableIndexed<String>()
{
@Override
public int size()
{
return bitmapIndex.getCardinality();
}
@Override
public String get(int index)
{
return bitmapIndex.getValue(index);
}
@Override
public int indexOf(String value)
{
return bitmapIndex.getIndex(value);
}
@Override
public Iterator<String> iterator()
{
return IndexedIterable.create(this).iterator();
}
@Override
public void inspectRuntimeShape(RuntimeShapeInspector inspector)
{
inspector.visit("column", bitmapIndex);
}
@Override
public void close()
{
}
};
}
final ColumnHolder columnHolder = index.getColumnHolder(dimension);
if (columnHolder == null) {
return null;
}
if (!columnHolder.getCapabilities().toColumnType().is(ValueType.STRING)) {
// this method only supports returning STRING values, so other types of dictionary encoded columns will not
// work correctly here until reworking is done to support filtering/indexing other types of columns
return null;
}
BaseColumn col = columnHolder.getColumn();
if (!(col instanceof DictionaryEncodedColumn)) {
return null;
}
final DictionaryEncodedColumn<String> column = (DictionaryEncodedColumn<String>) col;
return new CloseableIndexed<String>()
{
@Override
public int size()
{
return column.getCardinality();
}
@Override
public String get(int index)
{
return column.lookupName(index);
}
@Override
public int indexOf(String value)
{
return column.lookupId(value);
}
@Override
public Iterator<String> iterator()
{
return IndexedIterable.create(this).iterator();
}
@Override
public void inspectRuntimeShape(RuntimeShapeInspector inspector)
{
inspector.visit("column", column);
}
@Override
public void close() throws IOException
{
column.close();
}
};
}
@Override
public ColumnCapabilities.Capable hasMultipleValues(final String dimension)
{
if (isVirtualColumn(dimension)) {
VirtualColumn virtualColumn = virtualColumns.getVirtualColumn(dimension);
ColumnCapabilities virtualCapabilities = null;
if (virtualColumn != null) {
virtualCapabilities = virtualColumn.capabilities(index, dimension);
}
return virtualCapabilities != null ? virtualCapabilities.hasMultipleValues() : ColumnCapabilities.Capable.FALSE;
}
final ColumnHolder columnHolder = index.getColumnHolder(dimension);
// if ColumnHolder is null, the column doesn't exist, but report as not having multiple values so that
// the empty bitmap will be used
return columnHolder != null
? columnHolder.getCapabilities().hasMultipleValues()
: ColumnCapabilities.Capable.FALSE;
}
@Override
public int getNumRows()
{
try (final NumericColumn column = (NumericColumn) index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getColumn()) {
return column.length();
}
}
@Override
public BitmapFactory getBitmapFactory()
{
return bitmapFactory;
}
@Override
@Nullable
public BitmapIndex getBitmapIndex(String dimension)
{
if (isVirtualColumn(dimension)) {
return virtualColumns.getBitmapIndex(dimension, index);
}
final ColumnHolder columnHolder = index.getColumnHolder(dimension);
if (columnHolder == null || !columnHolder.getCapabilities().isFilterable()) {
// for missing columns and columns with types that do not support filtering,
// treat the column as if it were a String column full of nulls.
// Create a BitmapIndex so that filters applied to null columns can use
// bitmap indexes. Filters check for the presence of a bitmap index, this is used to determine
// whether the filter is applied in the pre or post filtering stage.
return BitmapIndexes.forNilColumn(this::getNumRows, bitmapFactory);
} else if (columnHolder.getCapabilities().hasBitmapIndexes() && columnHolder.getCapabilities().is(ValueType.STRING)) {
// currently BitmapIndex are reliant on STRING dictionaries to operate correctly, and will fail when used with
// other types of dictionary encoded columns
return columnHolder.getBitmapIndex();
} else {
return null;
}
}
@Override
public ImmutableBitmap getBitmapIndex(String dimension, String value)
{
if (isVirtualColumn(dimension)) {
BitmapIndex idx = virtualColumns.getBitmapIndex(dimension, index);
if (idx == null) {
return null;
}
return idx.getBitmapForValue(value);
}
final ColumnHolder columnHolder = index.getColumnHolder(dimension);
if (columnHolder == null || !columnHolder.getCapabilities().isFilterable()) {
if (NullHandling.isNullOrEquivalent(value)) {
return bitmapFactory.complement(bitmapFactory.makeEmptyImmutableBitmap(), getNumRows());
} else {
return bitmapFactory.makeEmptyImmutableBitmap();
}
}
if (!columnHolder.getCapabilities().hasBitmapIndexes() || !columnHolder.getCapabilities().is(ValueType.STRING)) {
// currently BitmapIndex are reliant on STRING dictionaries to operate correctly, and will fail when used with
// other types of dictionary encoded columns
return null;
}
final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
return bitmapIndex.getBitmapForValue(value);
}
@Override
public ImmutableRTree getSpatialIndex(String dimension)
{
if (isVirtualColumn(dimension)) {
return ImmutableRTree.empty();
}
final ColumnHolder columnHolder = index.getColumnHolder(dimension);
if (columnHolder == null || !columnHolder.getCapabilities().hasSpatialIndexes()) {
return ImmutableRTree.empty();
}
return columnHolder.getSpatialIndex().getRTree();
}
private boolean isVirtualColumn(final String columnName)
{
return virtualColumns.getVirtualColumn(columnName) != null;
}
@Nullable
@Override
public ColumnCapabilities getColumnCapabilities(String column)
{
return virtualColumns.getColumnCapabilities(index, column);
}
}

94
processing/src/main/java/org/apache/druid/segment/ColumnSelectorColumnIndexSelector.java Normal file
View File

@ -0,0 +1,94 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.NumericColumn;
import javax.annotation.Nullable;
/**
*/
public class ColumnSelectorColumnIndexSelector implements ColumnIndexSelector
{
private final BitmapFactory bitmapFactory;
private final VirtualColumns virtualColumns;
private final ColumnSelector columnSelector;
public ColumnSelectorColumnIndexSelector(
final BitmapFactory bitmapFactory,
final VirtualColumns virtualColumns,
final ColumnSelector index
)
{
this.bitmapFactory = bitmapFactory;
this.virtualColumns = virtualColumns;
this.columnSelector = index;
}
@Override
public int getNumRows()
{
try (final NumericColumn column = (NumericColumn) columnSelector.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getColumn()) {
return column.length();
}
}
@Override
public BitmapFactory getBitmapFactory()
{
return bitmapFactory;
}
@Override
public ColumnIndexSupplier getIndexSupplier(String column)
{
final ColumnIndexSupplier indexSupplier;
if (isVirtualColumn(column)) {
indexSupplier = virtualColumns.getIndexSupplier(column, columnSelector);
} else {
final ColumnHolder columnHolder = columnSelector.getColumnHolder(column);
// for missing columns and columns with types that do not support filtering,
// treat the column as if it were full of nulls. This allows callers to fabricate an 'all true' or 'all false'
// index so that filters which match the values can still use "indexes".
if (columnHolder == null || !columnHolder.getCapabilities().isFilterable()) {
return null;
}
indexSupplier = columnHolder.getIndexSupplier();
}
return indexSupplier;
}
private boolean isVirtualColumn(final String columnName)
{
return virtualColumns.getVirtualColumn(columnName) != null;
}
@Nullable
@Override
public ColumnCapabilities getColumnCapabilities(String column)
{
return virtualColumns.getColumnCapabilities(columnSelector, column);
}
}

11
processing/src/main/java/org/apache/druid/segment/FilteredOffset.java
View File

@ -21,11 +21,13 @@ package org.apache.druid.segment;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.query.BaseQuery;
import org.apache.druid.query.DefaultBitmapResultFactory;
import org.apache.druid.query.filter.BooleanFilter;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.RowOffsetMatcherFactory;
import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.data.Offset;
import org.apache.druid.segment.data.ReadableOffset;
import org.apache.druid.segment.filter.BooleanValueMatcher;
@ -41,7 +43,7 @@ public final class FilteredOffset extends Offset
ColumnSelectorFactory columnSelectorFactory,
boolean descending,
Filter postFilter,
ColumnSelectorBitmapIndexSelector bitmapIndexSelector
ColumnSelectorColumnIndexSelector bitmapIndexSelector
)
{
this.baseOffset = baseOffset;
@ -56,9 +58,12 @@ public final class FilteredOffset extends Offset
rowOffsetMatcherFactory
);
} else {
if (postFilter.shouldUseBitmapIndex(bitmapIndexSelector)) {
final BitmapColumnIndex columnIndex = postFilter.getBitmapColumnIndex(bitmapIndexSelector);
// we only consider "exact" indexes here, because if false, we've already used the bitmap index for the base
// offset and must use the value matcher here
if (columnIndex != null && columnIndex.getIndexCapabilities().isExact()) {
filterMatcher = rowOffsetMatcherFactory.makeRowOffsetMatcher(
postFilter.getBitmapIndex(bitmapIndexSelector)
columnIndex.computeBitmapResult(new DefaultBitmapResultFactory(bitmapIndexSelector.getBitmapFactory()))
);
} else {
filterMatcher = postFilter.makeMatcher(columnSelectorFactory);

25
processing/src/main/java/org/apache/druid/segment/IndexIO.java
View File

@ -64,10 +64,9 @@ import org.apache.druid.segment.data.IndexedIterable;
import org.apache.druid.segment.data.VSizeColumnarMultiInts;
import org.apache.druid.segment.serde.ComplexColumnPartSupplier;
import org.apache.druid.segment.serde.DictionaryEncodedColumnSupplier;
import org.apache.druid.segment.serde.DictionaryEncodedStringIndexSupplier;
import org.apache.druid.segment.serde.FloatNumericColumnSupplier;
import org.apache.druid.segment.serde.LongNumericColumnSupplier;
import org.apache.druid.segment.serde.SpatialIndexColumnPartSupplier;
import org.apache.druid.segment.serde.StringBitmapIndexColumnPartSupplier;
import org.joda.time.Interval;
import javax.annotation.Nullable;
@ -454,17 +453,19 @@ public class IndexIO
Suppliers.ofInstance(index.getDimColumn(dimension)),
columnConfig.columnCacheSizeBytes()
)
)
.setBitmapIndex(
new StringBitmapIndexColumnPartSupplier(
new ConciseBitmapFactory(),
index.getBitmapIndexes().get(dimension),
index.getDimValueLookup(dimension)
)
);
if (index.getSpatialIndexes().get(dimension) != null) {
builder.setSpatialIndex(new SpatialIndexColumnPartSupplier(index.getSpatialIndexes().get(dimension)));
}
GenericIndexed<ImmutableBitmap> bitmaps = index.getBitmapIndexes().get(dimension);
ImmutableRTree spatialIndex = index.getSpatialIndexes().get(dimension);
builder.setIndexSupplier(
new DictionaryEncodedStringIndexSupplier(
new ConciseBitmapFactory(),
index.getDimValueLookup(dimension),
bitmaps,
spatialIndex
),
bitmaps != null,
spatialIndex != null
);
columns.put(dimension, getColumnHolderSupplier(builder, lazy));
}

2
processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java
View File

@ -337,7 +337,7 @@ public class IndexMergerV9 implements IndexMerger
ColumnDescriptor columnDesc = ColumnDescriptor
.builder()
.setValueType(dimCapabilities.get(i).getType())
.addSerde(new NullColumnPartSerde(indexMergeResult.rowCount, indexSpec.getBitmapSerdeFactory()))
.addSerde(new NullColumnPartSerde(indexMergeResult.rowCount))
.build();
makeColumn(v9Smoosher, mergedDimensions.get(i), columnDesc);
}

12
processing/src/main/java/org/apache/druid/segment/IntIteratorUtils.java
View File

@ -19,11 +19,8 @@
package org.apache.druid.segment;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntIterator;
import it.unimi.dsi.fastutil.ints.IntIterators;
import it.unimi.dsi.fastutil.ints.IntList;
import it.unimi.dsi.fastutil.ints.IntLists;
import it.unimi.dsi.fastutil.longs.LongHeaps;
import org.apache.druid.java.util.common.IAE;
@ -196,15 +193,6 @@ public final class IntIteratorUtils
}
}
public static IntList toIntList(IntIterator iterator)
{
final IntList integers = new IntArrayList();
while (iterator.hasNext()) {
integers.add(iterator.nextInt());
}
return IntLists.unmodifiable(integers);
}
private IntIteratorUtils()
{
}

4
processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorSequenceBuilder.java
View File

@ -66,7 +66,7 @@ public class QueryableIndexCursorSequenceBuilder
@Nullable
private final Filter postFilter;
@Nullable
private final ColumnSelectorBitmapIndexSelector bitmapIndexSelector;
private final ColumnSelectorColumnIndexSelector bitmapIndexSelector;
public QueryableIndexCursorSequenceBuilder(
QueryableIndex index,
@ -77,7 +77,7 @@ public class QueryableIndexCursorSequenceBuilder
long maxDataTimestamp,
boolean descending,
@Nullable Filter postFilter,
@Nullable ColumnSelectorBitmapIndexSelector bitmapIndexSelector
@Nullable ColumnSelectorColumnIndexSelector bitmapIndexSelector
)
{
this.index = index;

32
processing/src/main/java/org/apache/druid/segment/QueryableIndexIndexableAdapter.java
View File

@ -19,18 +19,18 @@
package org.apache.druid.segment;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.column.BaseColumn;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.ComplexColumn;
import org.apache.druid.segment.column.DictionaryEncodedColumn;
import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex;
import org.apache.druid.segment.data.BitmapValues;
import org.apache.druid.segment.data.CloseableIndexed;
import org.apache.druid.segment.data.ImmutableBitmapValues;
@ -68,6 +68,11 @@ public class QueryableIndexIndexableAdapter implements IndexableAdapter
this.metadata = input.getMetadata();
}
public QueryableIndex getQueryableIndex()
{
return input;
}
@Override
public Interval getDataInterval()
{
@ -371,7 +376,11 @@ public class QueryableIndexIndexableAdapter implements IndexableAdapter
return BitmapValues.EMPTY;
}
final BitmapIndex bitmaps = columnHolder.getBitmapIndex();
final ColumnIndexSupplier indexSupplier = columnHolder.getIndexSupplier();
if (indexSupplier == null) {
return BitmapValues.EMPTY;
}
final DictionaryEncodedStringValueIndex bitmaps = indexSupplier.as(DictionaryEncodedStringValueIndex.class);
if (bitmaps == null) {
return BitmapValues.EMPTY;
}
@ -383,23 +392,6 @@ public class QueryableIndexIndexableAdapter implements IndexableAdapter
}
}
@VisibleForTesting
BitmapValues getBitmapIndex(String dimension, String value)
{
final ColumnHolder columnHolder = input.getColumnHolder(dimension);
if (columnHolder == null) {
return BitmapValues.EMPTY;
}
final BitmapIndex bitmaps = columnHolder.getBitmapIndex();
if (bitmaps == null) {
return BitmapValues.EMPTY;
}
return new ImmutableBitmapValues(bitmaps.getBitmapForValue(value));
}
@Override
public Metadata getMetadata()
{

51
processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java
View File

@ -33,10 +33,12 @@ import org.apache.druid.query.DefaultBitmapResultFactory;
import org.apache.druid.query.QueryMetrics;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.segment.column.BaseColumn;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.DictionaryEncodedColumn;
import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex;
import org.apache.druid.segment.column.NumericColumn;
import org.apache.druid.segment.data.Indexed;
import org.apache.druid.segment.filter.AndFilter;
@ -146,8 +148,9 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
{
ColumnHolder columnHolder = index.getColumnHolder(dimension);
if (columnHolder != null && columnHolder.getCapabilities().hasBitmapIndexes()) {
BitmapIndex bitmap = columnHolder.getBitmapIndex();
return bitmap.getCardinality() > 0 ? bitmap.getValue(0) : null;
ColumnIndexSupplier indexSupplier = columnHolder.getIndexSupplier();
DictionaryEncodedStringValueIndex index = indexSupplier.as(DictionaryEncodedStringValueIndex.class);
return index.getCardinality() > 0 ? index.getValue(0) : null;
}
return null;
}
@ -158,8 +161,9 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
{
ColumnHolder columnHolder = index.getColumnHolder(dimension);
if (columnHolder != null && columnHolder.getCapabilities().hasBitmapIndexes()) {
BitmapIndex bitmap = columnHolder.getBitmapIndex();
return bitmap.getCardinality() > 0 ? bitmap.getValue(bitmap.getCardinality() - 1) : null;
ColumnIndexSupplier indexSupplier = columnHolder.getIndexSupplier();
DictionaryEncodedStringValueIndex index = indexSupplier.as(DictionaryEncodedStringValueIndex.class);
return index.getCardinality() > 0 ? index.getValue(index.getCardinality() - 1) : null;
}
return null;
}
@ -187,7 +191,8 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
{
if (filter != null) {
final boolean filterCanVectorize =
filter.shouldUseBitmapIndex(makeBitmapIndexSelector(virtualColumns)) || filter.canVectorizeMatcher(this);
filter.getBitmapColumnIndex(makeBitmapIndexSelector(virtualColumns)) != null
|| filter.canVectorizeMatcher(this);
if (!filterCanVectorize) {
return false;
@ -223,7 +228,7 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
return null;
}
final ColumnSelectorBitmapIndexSelector bitmapIndexSelector = makeBitmapIndexSelector(virtualColumns);
final ColumnSelectorColumnIndexSelector bitmapIndexSelector = makeBitmapIndexSelector(virtualColumns);
final FilterAnalysis filterAnalysis = analyzeFilter(filter, bitmapIndexSelector, queryMetrics);
@ -260,7 +265,7 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
return Sequences.empty();
}
final ColumnSelectorBitmapIndexSelector bitmapIndexSelector = makeBitmapIndexSelector(virtualColumns);
final ColumnSelectorColumnIndexSelector bitmapIndexSelector = makeBitmapIndexSelector(virtualColumns);
final FilterAnalysis filterAnalysis = analyzeFilter(filter, bitmapIndexSelector, queryMetrics);
@ -311,9 +316,9 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
}
@VisibleForTesting
public ColumnSelectorBitmapIndexSelector makeBitmapIndexSelector(final VirtualColumns virtualColumns)
public ColumnSelectorColumnIndexSelector makeBitmapIndexSelector(final VirtualColumns virtualColumns)
{
return new ColumnSelectorBitmapIndexSelector(
return new ColumnSelectorColumnIndexSelector(
index.getBitmapFactoryForDimensions(),
virtualColumns,
index
@ -323,7 +328,7 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
@VisibleForTesting
public FilterAnalysis analyzeFilter(
@Nullable final Filter filter,
ColumnSelectorBitmapIndexSelector indexSelector,
ColumnSelectorColumnIndexSelector indexSelector,
@Nullable QueryMetrics queryMetrics
)
{
@ -338,7 +343,7 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
* were not pruned AND those that matched the filter during row scanning)
*
* An AND filter can have its subfilters partitioned across the two steps. The subfilters that can be
* processed entirely with bitmap indexes (subfilter returns true for supportsBitmapIndex())
* processed entirely with bitmap indexes (subfilter returns non-null value for getBitmapColumnIndex)
* will be moved to the pre-filtering stage.
*
* Any subfilters that cannot be processed entirely with bitmap indexes will be moved to the post-filtering stage.
@ -355,19 +360,27 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
// If we get an AndFilter, we can split the subfilters across both filtering stages
for (Filter subfilter : ((AndFilter) filter).getFilters()) {
if (subfilter.supportsBitmapIndex(indexSelector) && subfilter.shouldUseBitmapIndex(indexSelector)) {
final BitmapColumnIndex columnIndex = subfilter.getBitmapColumnIndex(indexSelector);
preFilters.add(subfilter);
} else {
if (columnIndex == null) {
postFilters.add(subfilter);
} else {
preFilters.add(subfilter);
if (!columnIndex.getIndexCapabilities().isExact()) {
postFilters.add(subfilter);
}
}
}
} else {
// If we get an OrFilter or a single filter, handle the filter in one stage
if (filter.supportsBitmapIndex(indexSelector) && filter.shouldUseBitmapIndex(indexSelector)) {
preFilters.add(filter);
} else {
final BitmapColumnIndex columnIndex = filter.getBitmapColumnIndex(indexSelector);
if (columnIndex == null) {
postFilters.add(filter);
} else {
preFilters.add(filter);
if (!columnIndex.getIndexCapabilities().isExact()) {
postFilters.add(filter);
}
}
}
}
@ -380,7 +393,7 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
BitmapResultFactory<?> bitmapResultFactory =
queryMetrics.makeBitmapResultFactory(indexSelector.getBitmapFactory());
long bitmapConstructionStartNs = System.nanoTime();
// Use AndFilter.getBitmapResult to intersect the preFilters to get its short-circuiting behavior.
// Use AndFilter.getBitmapIndex to intersect the preFilters to get its short-circuiting behavior.
preFilterBitmap = AndFilter.getBitmapIndex(indexSelector, bitmapResultFactory, preFilters);
preFilteredRows = preFilterBitmap.size();
queryMetrics.reportBitmapConstructionTime(System.nanoTime() - bitmapConstructionStartNs);

16
processing/src/main/java/org/apache/druid/segment/VirtualColumn.java
View File

@ -23,9 +23,10 @@ import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import org.apache.druid.java.util.common.Cacheable;
import org.apache.druid.query.dimension.DimensionSpec;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.data.ReadableOffset;
import org.apache.druid.segment.serde.NoIndexesColumnIndexSupplier;
import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector;
import org.apache.druid.segment.vector.ReadableVectorOffset;
import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector;
@ -294,16 +295,15 @@ public interface VirtualColumn extends Cacheable
boolean usesDotNotation();
/**
* Returns the BitmapIndex for efficient filtering on columns that support it. This method is only used if
* {@link ColumnCapabilities} returned from {@link #capabilities(String)} has flag for BitmapIndex support.
* @param columnName
* @param selector
* @return BitmapIndex
* Get the {@link ColumnIndexSupplier} for the specified virtual column, with the assistance of a
* {@link ColumnSelector} to allow reading things from segments. If the virtual column has no indexes, this method
* will return null, or may also return a non-null supplier whose methods may return null values - having a supplier
* is no guarantee that the column has indexes.
*/
@SuppressWarnings("unused")
@Nullable
default BitmapIndex getBitmapIndex(String columnName, ColumnSelector selector)
default ColumnIndexSupplier getIndexSupplier(String columnName, ColumnSelector columnSelector)
{
throw new UnsupportedOperationException("not supported");
return NoIndexesColumnIndexSupplier.getInstance();
}
}

15
processing/src/main/java/org/apache/druid/segment/VirtualColumns.java
View File

@ -33,9 +33,9 @@ import org.apache.druid.query.Query;
import org.apache.druid.query.QueryContexts;
import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.query.dimension.DimensionSpec;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.data.ReadableOffset;
import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector;
import org.apache.druid.segment.vector.ReadableVectorOffset;
@ -170,13 +170,17 @@ public class VirtualColumns implements Cacheable
return withDotSupport.get(baseColumnName);
}
/**
* Get the {@link ColumnIndexSupplier} of the specified virtual column, with the assistance of a
* {@link ColumnSelector} to allow reading things from segments. If the column does not have indexes this method
* may return null, or may also return a non-null supplier whose methods may return null values - having a supplier
* is no guarantee that the column has indexes.
*/
@Nullable
public BitmapIndex getBitmapIndex(String columnName, ColumnSelector columnSelector)
public ColumnIndexSupplier getIndexSupplier(String columnName, ColumnSelector columnSelector)
{
final VirtualColumn virtualColumn = getVirtualColumnForSelector(columnName);
return virtualColumn.capabilities(columnSelector, columnName).hasBitmapIndexes()
? virtualColumn.getBitmapIndex(columnName, columnSelector)
: null;
return virtualColumn.getIndexSupplier(columnName, columnSelector);
}
/**
@ -497,4 +501,5 @@ public class VirtualColumns implements Cacheable
{
return virtualColumns.toString();
}
}

51
processing/src/main/java/org/apache/druid/segment/column/AllFalseBitmapColumnIndex.java Normal file
View File

@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment.column;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.ColumnIndexSelector;
public class AllFalseBitmapColumnIndex implements BitmapColumnIndex
{
private final ColumnIndexSelector selector;
public AllFalseBitmapColumnIndex(ColumnIndexSelector indexSelector)
{
this.selector = indexSelector;
}
@Override
public ColumnIndexCapabilities getIndexCapabilities()
{
return SimpleColumnIndexCapabilities.getConstant();
}
@Override
public double estimateSelectivity(int totalRows)
{
return 0;
}
@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory)
{
return bitmapResultFactory.wrapAllFalse(selector.getBitmapFactory().makeEmptyImmutableBitmap());
}
}

54
processing/src/main/java/org/apache/druid/segment/column/AllTrueBitmapColumnIndex.java Normal file
View File

@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment.column;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.ColumnIndexSelector;
public class AllTrueBitmapColumnIndex implements BitmapColumnIndex
{
private final ColumnIndexSelector selector;
public AllTrueBitmapColumnIndex(ColumnIndexSelector indexSelector)
{
this.selector = indexSelector;
}
@Override
public ColumnIndexCapabilities getIndexCapabilities()
{
return SimpleColumnIndexCapabilities.getConstant();
}
@Override
public double estimateSelectivity(int totalRows)
{
return 1;
}
@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory)
{
return bitmapResultFactory.wrapAllTrue(
selector.getBitmapFactory()
.complement(selector.getBitmapFactory().makeEmptyImmutableBitmap(), selector.getNumRows())
);
}
}

31
processing/src/main/java/org/apache/druid/segment/column/BitmapColumnIndex.java Normal file
View File

@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment.column;
import org.apache.druid.query.BitmapResultFactory;
public interface BitmapColumnIndex
{
ColumnIndexCapabilities getIndexCapabilities();
double estimateSelectivity(int totalRows);
<T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory);
}

107
processing/src/main/java/org/apache/druid/segment/column/BitmapIndex.java
View File

@ -1,107 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment.column;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import javax.annotation.Nullable;
import java.util.Set;
import java.util.function.Predicate;
/**
* Provides a mechanism to get {@link ImmutableBitmap} for a value, or {@link Iterable} of {@link ImmutableBitmap}
* for a range or exact set of values, the set bits of which correspond to which rows contain the matching values.
*
* Used to power {@link org.apache.druid.segment.BitmapOffset} and
* {@link org.apache.druid.segment.vector.BitmapVectorOffset} which are used with column cursors for fast filtering
* of indexed values.
*
* The column must be ingested with a bitmap index for filters to use them and to participate in this "pre-filtering"
* step when scanning segments
*
* @see org.apache.druid.segment.QueryableIndexStorageAdapter#analyzeFilter
*/
public interface BitmapIndex
{
/**
* Get the cardinality of the underlying value dictionary
*/
int getCardinality();
/**
* Get the value in the underlying value dictionary of the specified dictionary id
*/
@Nullable
String getValue(int index);
/**
* Returns true if the underlying value dictionary has nulls
*/
boolean hasNulls();
BitmapFactory getBitmapFactory();
/**
* Returns the index of "value" in this BitmapIndex, or a negative value, if not present in the underlying dictionary
*/
int getIndex(@Nullable String value);
/**
* Get the {@link ImmutableBitmap} for dictionary id of the underlying dictionary
*/
ImmutableBitmap getBitmap(int idx);
/**
* Get the {@link ImmutableBitmap} corresponding to the supplied value
*/
ImmutableBitmap getBitmapForValue(@Nullable String value);
/**
* Get an {@link Iterable} of {@link ImmutableBitmap} corresponding to the values supplied in the specified range
*/
default Iterable<ImmutableBitmap> getBitmapsInRange(
@Nullable String startValue,
boolean startStrict,
@Nullable String endValue,
boolean endStrict
)
{
return getBitmapsInRange(startValue, startStrict, endValue, endStrict, (index) -> true);
}
/**
* Get an {@link Iterable} of {@link ImmutableBitmap} corresponding to the values supplied in the specified range
* whose dictionary ids also match some predicate
*/
Iterable<ImmutableBitmap> getBitmapsInRange(
@Nullable String startValue,
boolean startStrict,
@Nullable String endValue,
boolean endStrict,
Predicate<String> matcher
);
/**
* Get an {@link Iterable} of {@link ImmutableBitmap} corresponding to the specified set of values (if they are
* contained in the underlying column)
*/
Iterable<ImmutableBitmap> getBitmapsForValues(Set<String> values);
}

170
processing/src/main/java/org/apache/druid/segment/column/BitmapIndexes.java
View File

@ -1,170 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment.column;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.segment.serde.StringBitmapIndexColumnPartSupplier;
import javax.annotation.Nullable;
import java.util.Collections;
import java.util.Set;
import java.util.function.IntSupplier;
import java.util.function.Predicate;
public final class BitmapIndexes
{
/**
* Returns a bitmapIndex for null-only columns.
*
* @param rowCountSupplier a supplier that returns the row count of the segment that the column belongs to.
* Getting from the supplier the row count can be expensive and thus should be
* evaluated lazily.
* @param bitmapFactory a bitmapFactory to create a bitmapIndex.
*/
public static BitmapIndex forNilColumn(IntSupplier rowCountSupplier, BitmapFactory bitmapFactory)
{
return new BitmapIndex()
{
private final Supplier<ImmutableBitmap> nullBitmapSupplier = Suppliers.memoize(
() -> getBitmapFactory().complement(
getBitmapFactory().makeEmptyImmutableBitmap(),
rowCountSupplier.getAsInt()
)
);
@Override
public int getCardinality()
{
return 1;
}
@Nullable
@Override
public String getValue(int index)
{
return null;
}
@Override
public boolean hasNulls()
{
return true;
}
@Override
public BitmapFactory getBitmapFactory()
{
return bitmapFactory;
}
/**
* Return -2 for non-null values to match what the {@link BitmapIndex} implementation in
* {@link StringBitmapIndexColumnPartSupplier}
* would return for {@link BitmapIndex#getIndex(String)} when there is only a single index, for the null value.
* i.e., return an 'insertion point' of 1 for non-null values (see {@link BitmapIndex} interface)
*/
@Override
public int getIndex(@Nullable String value)
{
return NullHandling.isNullOrEquivalent(value) ? 0 : -2;
}
@Override
public ImmutableBitmap getBitmap(int idx)
{
if (idx == 0) {
return nullBitmapSupplier.get();
} else {
return bitmapFactory.makeEmptyImmutableBitmap();
}
}
@Override
public ImmutableBitmap getBitmapForValue(@Nullable String value)
{
if (NullHandling.isNullOrEquivalent(value)) {
return nullBitmapSupplier.get();
} else {
return bitmapFactory.makeEmptyImmutableBitmap();
}
}
@Override
public Iterable<ImmutableBitmap> getBitmapsInRange(
@Nullable String startValue,
boolean startStrict,
@Nullable String endValue,
boolean endStrict,
Predicate<String> matcher
)
{
final int startIndex; // inclusive
int endIndex; // exclusive
if (startValue == null) {
startIndex = 0;
} else {
if (NullHandling.isNullOrEquivalent(startValue)) {
startIndex = startStrict ? 1 : 0;
} else {
startIndex = 1;
}
}
if (endValue == null) {
endIndex = 1;
} else {
if (NullHandling.isNullOrEquivalent(endValue)) {
endIndex = endStrict ? 0 : 1;
} else {
endIndex = 1;
}
}
endIndex = Math.max(startIndex, endIndex);
if (startIndex == endIndex) {
return Collections.emptyList();
}
if (matcher.test(null)) {
return ImmutableList.of(getBitmap(0));
}
return ImmutableList.of(bitmapFactory.makeEmptyImmutableBitmap());
}
@Override
public Iterable<ImmutableBitmap> getBitmapsForValues(Set<String> values)
{
if (values.contains(null) || (NullHandling.replaceWithDefault() && values.contains(""))) {
return ImmutableList.of(getBitmap(0));
}
return ImmutableList.of(bitmapFactory.makeEmptyImmutableBitmap());
}
};
}
private BitmapIndexes()
{
}
}

25
processing/src/main/java/org/apache/druid/segment/column/ColumnBuilder.java
View File

@ -22,6 +22,7 @@ package org.apache.druid.segment.column;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import org.apache.druid.java.util.common.io.smoosh.SmooshedFileMapper;
import org.apache.druid.segment.serde.NoIndexesColumnIndexSupplier;
import javax.annotation.Nullable;
@ -34,9 +35,7 @@ public class ColumnBuilder
@Nullable
private Supplier<? extends BaseColumn> columnSupplier = null;
@Nullable
private Supplier<BitmapIndex> bitmapIndex = null;
@Nullable
private Supplier<SpatialIndex> spatialIndex = null;
private ColumnIndexSupplier indexSupplier = NoIndexesColumnIndexSupplier.getInstance();
@Nullable
private SmooshedFileMapper fileMapper = null;
@ -98,17 +97,15 @@ public class ColumnBuilder
return this;
}
public ColumnBuilder setBitmapIndex(Supplier<BitmapIndex> bitmapIndex)
public ColumnBuilder setIndexSupplier(
@Nullable ColumnIndexSupplier indexSupplier,
boolean hasBitmapIndex,
boolean hasSpatial
)
{
this.bitmapIndex = bitmapIndex;
this.capabilitiesBuilder.setHasBitmapIndexes(true);
return this;
}
public ColumnBuilder setSpatialIndex(Supplier<SpatialIndex> spatialIndex)
{
this.spatialIndex = spatialIndex;
this.capabilitiesBuilder.setHasSpatialIndexes(true);
this.indexSupplier = indexSupplier;
capabilitiesBuilder.setHasBitmapIndexes(hasBitmapIndex);
capabilitiesBuilder.setHasSpatialIndexes(hasSpatial);
return this;
}
@ -127,6 +124,6 @@ public class ColumnBuilder
{
Preconditions.checkState(capabilitiesBuilder.getType() != null, "Type must be set.");
return new SimpleColumnHolder(capabilitiesBuilder, columnSupplier, bitmapIndex, spatialIndex);
return new SimpleColumnHolder(capabilitiesBuilder, columnSupplier, indexSupplier);
}
}

5
processing/src/main/java/org/apache/druid/segment/column/ColumnHolder.java
View File

@ -41,10 +41,9 @@ public interface ColumnHolder
int getLength();
BaseColumn getColumn();
@Nullable
BitmapIndex getBitmapIndex();
@Nullable
SpatialIndex getSpatialIndex();
ColumnIndexSupplier getIndexSupplier();
/**
* Returns a new instance of a {@link SettableColumnValueSelector}, corresponding to the type of this column.

43
processing/src/main/java/org/apache/druid/segment/column/ColumnIndexCapabilities.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment.column;
import org.apache.druid.query.filter.ColumnIndexSelector;
/**
* Sort of like {@link ColumnCapabilities}, except for indexes supplied by {@link ColumnIndexSelector}, provides
* information for how query processing may use indexes.
*/
public interface ColumnIndexCapabilities
{
/**
* Indicates if an index can be inverted for use with a 'NOT' filter. Some types of indexes may not be invertible,
* such as those which provide false positive matches.
*/
boolean isInvertible();
/**
* Indicates if an index is an exact match, or should also be post-filtered with a value matcher. Filters which
* are not an exact match must always use a value matcher as a post-filter, even if they have an index.
*/
boolean isExact();
ColumnIndexCapabilities merge(ColumnIndexCapabilities other);
}

45
processing/src/main/java/org/apache/druid/segment/column/ColumnIndexSupplier.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment.column;
import javax.annotation.Nullable;
/**
* Provides indexes and information about them ({@link ColumnIndexCapabilities}) for a column. Indexes which satisfy
* the {@link org.apache.druid.query.filter.Filter} used in a {@link org.apache.druid.query.Query}, allow the query
* engine to construct {@link org.apache.druid.segment.data.Offset} (and
* {@link org.apache.druid.segment.vector.VectorOffset}) to build {@link org.apache.druid.segment.Cursor}
* (and {@link org.apache.druid.segment.vector.VectorCursor}). This allows the engine to only scan the rows which match
* the filter values, instead of performing a full scan of the column and using a
* {@link org.apache.druid.query.filter.ValueMatcher}
* (or {@link org.apache.druid.query.filter.vector.VectorValueMatcher}) to filter the values.
*/
public interface ColumnIndexSupplier
{
/**
* Try to get a column 'index' of the specified type. If the index of the desired type is not available, this method
* will return null. If the value is non-null, the index may be used for the eventual construction of an
* {@link org.apache.druid.segment.data.Offset} to form the basis of a {@link org.apache.druid.segment.Cursor}
* (or {@link org.apache.druid.segment.vector.VectorOffset} and {@link org.apache.druid.segment.vector.VectorCursor})
* which can greatly reduce the total number of rows which need to be scanned and processed.
*/
@Nullable
<T> T as(Class<T> clazz);
}

59
processing/src/main/java/org/apache/druid/segment/column/DictionaryEncodedStringValueIndex.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment.column;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import javax.annotation.Nullable;
/**
* This exposes a 'raw' view into a bitmap value indexes of a string {@link DictionaryEncodedColumn}, allowing
* operation via dictionaryIds, as well as access to lower level details of such a column like value lookup and
* value cardinality.
*
* This interface should only be used when it is beneficial to operate in such a manner, most filter implementations
* should likely be using {@link StringValueSetIndex}, {@link DruidPredicateIndex}, {@link LexicographicalRangeIndex} or
* some other higher level index instead.
*/
public interface DictionaryEncodedStringValueIndex
{
boolean hasNulls();
/**
* Get the cardinality of the underlying value dictionary
*/
int getCardinality();
/**
* Get the value in the underlying value dictionary of the specified dictionary id
*/
@Nullable
String getValue(int index);
/**
* Returns the index of "value" in this DictionaryEncodedStringValueIndex, or a negative value, if not present in
* the underlying dictionary
*/
int getIndex(@Nullable String value);
/**
* Get the {@link ImmutableBitmap} for dictionary id of the underlying dictionary
*/
ImmutableBitmap getBitmap(int idx);
}

30
processing/src/main/java/org/apache/druid/segment/column/DruidPredicateIndex.java Normal file
View File

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment.column;
import org.apache.druid.query.filter.DruidPredicateFactory;
/**
* Uses a {@link DruidPredicateFactory} to construct a {@link BitmapColumnIndex}
*/
public interface DruidPredicateIndex
{
BitmapColumnIndex forPredicate(DruidPredicateFactory matcherFactory);
}

56
processing/src/main/java/org/apache/druid/segment/column/LexicographicalRangeIndex.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment.column;
import com.google.common.base.Predicate;
import javax.annotation.Nullable;
/**
* An optimized column value {@link BitmapColumnIndex} provider for columns which are stored in 'lexicographical' order,
* allowing short-circuit processing of string value ranges.
*/
public interface LexicographicalRangeIndex
{
/**
* Get an {@link BitmapColumnIndex} corresponding to the values supplied in the specified range
*/
default BitmapColumnIndex forRange(
@Nullable String startValue,
boolean startStrict,
@Nullable String endValue,
boolean endStrict
)
{
return forRange(startValue, startStrict, endValue, endStrict, (index) -> true);
}
/**
* Get an {@link BitmapColumnIndex} corresponding to the values supplied in the specified range whose dictionary ids
* also match some predicate, such as to match a prefix
*/
BitmapColumnIndex forRange(
@Nullable String startValue,
boolean startStrict,
@Nullable String endValue,
boolean endStrict,
Predicate<String> matcher
);
}

26
processing/src/main/java/org/apache/druid/segment/column/SimpleColumnHolder.java
View File

@ -28,27 +28,30 @@ import org.apache.druid.segment.selector.settable.SettableObjectColumnValueSelec
import javax.annotation.Nullable;
/**
*
*/
class SimpleColumnHolder implements ColumnHolder
{
private final ColumnCapabilities capabilities;
@Nullable
private final Supplier<? extends BaseColumn> columnSupplier;
@Nullable
private final Supplier<BitmapIndex> bitmapIndex;
@Nullable
private final Supplier<SpatialIndex> spatialIndex;
private final ColumnIndexSupplier indexSupplier;
private static final InvalidComplexColumnTypeValueSelector INVALID_COMPLEX_COLUMN_TYPE_VALUE_SELECTOR
= new InvalidComplexColumnTypeValueSelector();
SimpleColumnHolder(
ColumnCapabilities capabilities,
@Nullable Supplier<? extends BaseColumn> columnSupplier,
@Nullable Supplier<BitmapIndex> bitmapIndex,
@Nullable Supplier<SpatialIndex> spatialIndex
@Nullable ColumnIndexSupplier indexSupplier
)
{
this.capabilities = capabilities;
this.columnSupplier = columnSupplier;
this.indexSupplier = indexSupplier;
// ColumnSupplier being null is sort of a rare case but can happen when a segment
// was created, for example, using an aggregator that was removed in later versions.
// In such cases we are not able to deserialize the column metadata and determine
@ -62,8 +65,6 @@ class SimpleColumnHolder implements ColumnHolder
"Only complex column types can have nullable column suppliers"
);
}
this.bitmapIndex = bitmapIndex;
this.spatialIndex = spatialIndex;
}
@Override
@ -90,16 +91,9 @@ class SimpleColumnHolder implements ColumnHolder
@Nullable
@Override
public BitmapIndex getBitmapIndex()
public ColumnIndexSupplier getIndexSupplier()
{
return bitmapIndex == null ? null : bitmapIndex.get();
}
@Nullable
@Override
public SpatialIndex getSpatialIndex()
{
return spatialIndex == null ? null : spatialIndex.get();
return indexSupplier;
}
@Override

84
processing/src/main/java/org/apache/druid/segment/column/SimpleColumnIndexCapabilities.java Normal file
View File

@ -0,0 +1,84 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment.column;
import java.util.Objects;
public class SimpleColumnIndexCapabilities implements ColumnIndexCapabilities
{
private static final SimpleColumnIndexCapabilities DEFAULT = new SimpleColumnIndexCapabilities(true, true);
/**
* {@link ColumnIndexCapabilities} to use for constant values (including 'null' values for 'missing' columns)
*/
public static ColumnIndexCapabilities getConstant()
{
return DEFAULT;
}
private final boolean invertible;
private final boolean exact;
public SimpleColumnIndexCapabilities(boolean invertible, boolean exact)
{
this.invertible = invertible;
this.exact = exact;
}
@Override
public boolean isInvertible()
{
return invertible;
}
@Override
public boolean isExact()
{
return exact;
}
@Override
public ColumnIndexCapabilities merge(ColumnIndexCapabilities other)
{
return new SimpleColumnIndexCapabilities(
invertible && other.isInvertible(),
exact && other.isExact()
);
}
@Override
public boolean equals(Object o)
{
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
SimpleColumnIndexCapabilities that = (SimpleColumnIndexCapabilities) o;
return invertible == that.invertible && exact == that.exact;
}
@Override
public int hashCode()
{
return Objects.hash(invertible, exact);
}
}

42
processing/src/main/java/org/apache/druid/segment/column/StringValueSetIndex.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment.column;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import javax.annotation.Nullable;
import java.util.Set;
/**
* Index on individual values, and provides bitmaps for the rows which contain these values
*/
public interface StringValueSetIndex
{
/**
* Get the {@link ImmutableBitmap} corresponding to the supplied value
*/
BitmapColumnIndex forValue(@Nullable String value);
/**
* Get an {@link Iterable} of {@link ImmutableBitmap} corresponding to the specified set of values (if they are
* contained in the underlying column)
*/
BitmapColumnIndex forValues(Set<String> values);
}

102
processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java
View File

@ -23,12 +23,13 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.DefaultBitmapResultFactory;
import org.apache.druid.query.filter.BooleanFilter;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.RowOffsetMatcherFactory;
import org.apache.druid.query.filter.ValueMatcher;
@ -38,8 +39,12 @@ import org.apache.druid.query.filter.vector.VectorValueMatcher;
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnIndexCapabilities;
import org.apache.druid.segment.column.SimpleColumnIndexCapabilities;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
@ -68,44 +73,81 @@ public class AndFilter implements BooleanFilter
}
public static <T> ImmutableBitmap getBitmapIndex(
BitmapIndexSelector selector,
ColumnIndexSelector selector,
BitmapResultFactory<T> bitmapResultFactory,
List<Filter> filters
)
{
return bitmapResultFactory.toImmutableBitmap(getBitmapResult(selector, bitmapResultFactory, filters));
return bitmapResultFactory.toImmutableBitmap(
getBitmapIndex(selector, filters).computeBitmapResult(bitmapResultFactory)
);
}
private static <T> T getBitmapResult(
BitmapIndexSelector selector,
BitmapResultFactory<T> bitmapResultFactory,
private static BitmapColumnIndex getBitmapIndex(
ColumnIndexSelector selector,
Collection<Filter> filters
)
{
if (filters.size() == 1) {
return Iterables.getOnlyElement(filters).getBitmapResult(selector, bitmapResultFactory);
return Iterables.getOnlyElement(filters).getBitmapColumnIndex(selector);
}
final List<T> bitmapResults = Lists.newArrayListWithCapacity(filters.size());
final List<BitmapColumnIndex> bitmapColumnIndices = new ArrayList<>(filters.size());
ColumnIndexCapabilities merged = new SimpleColumnIndexCapabilities(true, true);
for (final Filter filter : filters) {
Preconditions.checkArgument(filter.supportsBitmapIndex(selector),
"Filter[%s] does not support bitmap index", filter
);
final T bitmapResult = filter.getBitmapResult(selector, bitmapResultFactory);
if (bitmapResultFactory.isEmpty(bitmapResult)) {
// Short-circuit.
return bitmapResultFactory.wrapAllFalse(Filters.allFalse(selector));
final BitmapColumnIndex index = filter.getBitmapColumnIndex(selector);
if (index == null) {
// all or nothing
return null;
}
bitmapResults.add(bitmapResult);
merged = merged.merge(index.getIndexCapabilities());
bitmapColumnIndices.add(index);
}
return bitmapResultFactory.intersection(bitmapResults);
final ColumnIndexCapabilities finalMerged = merged;
return new BitmapColumnIndex()
{
@Override
public ColumnIndexCapabilities getIndexCapabilities()
{
return finalMerged;
}
@Override
public double estimateSelectivity(int totalRows)
{
// Estimate selectivity with attribute value independence assumption
double selectivity = 1.0;
for (final Filter filter : filters) {
selectivity *= filter.estimateSelectivity(selector);
}
return selectivity;
}
@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory)
{
final List<T> bitmapResults = new ArrayList<>(bitmapColumnIndices.size());
for (final BitmapColumnIndex index : bitmapColumnIndices) {
final T bitmapResult = index.computeBitmapResult(bitmapResultFactory);
if (bitmapResultFactory.isEmpty(bitmapResult)) {
// Short-circuit.
return bitmapResultFactory.wrapAllFalse(
selector.getBitmapFactory().makeEmptyImmutableBitmap()
);
}
bitmapResults.add(bitmapResult);
}
return bitmapResultFactory.intersection(bitmapResults);
}
};
}
@Nullable
@Override
public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory)
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
return getBitmapResult(selector, bitmapResultFactory, filters);
return getBitmapIndex(selector, filters);
}
@Override
@ -140,17 +182,20 @@ public class AndFilter implements BooleanFilter
@Override
public ValueMatcher makeMatcher(
BitmapIndexSelector selector,
ColumnIndexSelector selector,
ColumnSelectorFactory columnSelectorFactory,
RowOffsetMatcherFactory rowOffsetMatcherFactory
)
{
final List<ValueMatcher> matchers = new ArrayList<>();
final List<ImmutableBitmap> bitmaps = new ArrayList<>();
final BitmapFactory bitmapFactory = selector.getBitmapFactory();
final DefaultBitmapResultFactory resultFactory = new DefaultBitmapResultFactory(bitmapFactory);
for (Filter filter : filters) {
if (filter.supportsBitmapIndex(selector)) {
bitmaps.add(filter.getBitmapIndex(selector));
final BitmapColumnIndex columnIndex = filter.getBitmapColumnIndex(selector);
if (columnIndex != null && columnIndex.getIndexCapabilities().isExact()) {
bitmaps.add(columnIndex.computeBitmapResult(resultFactory));
} else {
ValueMatcher matcher = filter.makeMatcher(columnSelectorFactory);
matchers.add(matcher);
@ -172,17 +217,6 @@ public class AndFilter implements BooleanFilter
return filters;
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
{
// Estimate selectivity with attribute value independence assumption
double selectivity = 1.0;
for (final Filter filter : filters) {
selectivity *= filter.estimateSelectivity(indexSelector);
}
return selectivity;
}
@Override
public String toString()
{

97
processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java
View File

@ -22,13 +22,11 @@ package org.apache.druid.segment.filter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
import com.google.common.base.Supplier;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.BoundDimFilter;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.DruidDoublePredicate;
import org.apache.druid.query.filter.DruidFloatPredicate;
import org.apache.druid.query.filter.DruidLongPredicate;
@ -43,9 +41,12 @@ import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnProcessors;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.LexicographicalRangeIndex;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import javax.annotation.Nullable;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@ -64,55 +65,30 @@ public class BoundFilter implements Filter
}
@Override
public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory)
@Nullable
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
if (supportShortCircuit()) {
final BitmapIndex bitmapIndex = selector.getBitmapIndex(boundDimFilter.getDimension());
if (bitmapIndex == null || bitmapIndex.getCardinality() == 0) {
if (doesMatchNull()) {
return bitmapResultFactory.wrapAllTrue(Filters.allTrue(selector));
} else {
return bitmapResultFactory.wrapAllFalse(Filters.allFalse(selector));
}
}
return bitmapResultFactory.unionDimensionValueBitmaps(getBitmapIterator(boundDimFilter, bitmapIndex));
} else {
return Filters.matchPredicate(
boundDimFilter.getDimension(),
selector,
bitmapResultFactory,
getPredicateFactory().makeStringPredicate()
);
if (!Filters.checkFilterTuningUseIndex(boundDimFilter.getDimension(), selector, filterTuning)) {
return null;
}
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
{
if (supportShortCircuit()) {
final BitmapIndex bitmapIndex = indexSelector.getBitmapIndex(boundDimFilter.getDimension());
if (bitmapIndex == null || bitmapIndex.getCardinality() == 0) {
return doesMatchNull() ? 1. : 0.;
final ColumnIndexSupplier indexSupplier = selector.getIndexSupplier(boundDimFilter.getDimension());
if (indexSupplier == null) {
return Filters.makeNullIndex(doesMatchNull(), selector);
}
return Filters.estimateSelectivity(
bitmapIndex.getBitmapsInRange(
boundDimFilter.getLower(),
boundDimFilter.isLowerStrict(),
boundDimFilter.getUpper(),
boundDimFilter.isUpperStrict()
).iterator(),
indexSelector.getNumRows()
final LexicographicalRangeIndex rangeIndex = indexSupplier.as(LexicographicalRangeIndex.class);
if (rangeIndex == null) {
// column
return null;
}
return rangeIndex.forRange(
boundDimFilter.getLower(),
boundDimFilter.isLowerStrict(),
boundDimFilter.getUpper(),
boundDimFilter.isUpperStrict()
);
} else {
return Filters.estimateSelectivity(
boundDimFilter.getDimension(),
indexSelector,
getPredicateFactory().makeStringPredicate()
);
return Filters.makePredicateIndex(boundDimFilter.getDimension(), selector, getPredicateFactory());
}
}
@ -145,19 +121,7 @@ public class BoundFilter implements Filter
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return selector.getBitmapIndex(boundDimFilter.getDimension()) != null;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return Filters.shouldUseBitmapIndex(this, selector, filterTuning);
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, ColumnIndexSelector indexSelector)
{
return Filters.supportsSelectivityEstimation(this, boundDimFilter.getDimension(), columnSelector, indexSelector);
}
@ -201,19 +165,6 @@ public class BoundFilter implements Filter
);
}
private static Iterable<ImmutableBitmap> getBitmapIterator(
final BoundDimFilter boundDimFilter,
final BitmapIndex bitmapIndex
)
{
return bitmapIndex.getBitmapsInRange(
boundDimFilter.getLower(),
boundDimFilter.isLowerStrict(),
boundDimFilter.getUpper(),
boundDimFilter.isUpperStrict()
);
}
private DruidPredicateFactory getPredicateFactory()
{
return new BoundDimFilterDruidPredicateFactory(extractionFn, boundDimFilter);

26
processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java
View File

@ -20,9 +20,8 @@
package org.apache.druid.segment.filter;
import com.google.common.base.Preconditions;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.dimension.DimensionSpec;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
@ -35,6 +34,7 @@ import org.apache.druid.segment.ColumnProcessors;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.DimensionSelector;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.data.IndexedInts;
@ -55,10 +55,11 @@ public class ColumnComparisonFilter implements Filter
this.dimensions = Preconditions.checkNotNull(dimensions, "dimensions");
}
@Nullable
@Override
public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory)
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
throw new UnsupportedOperationException();
return null;
}
@Override
@ -134,19 +135,7 @@ public class ColumnComparisonFilter implements Filter
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return false;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return false;
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, ColumnIndexSelector indexSelector)
{
return false;
}
@ -158,12 +147,11 @@ public class ColumnComparisonFilter implements Filter
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
public double estimateSelectivity(ColumnIndexSelector indexSelector)
{
throw new UnsupportedOperationException();
}
@Override
public boolean equals(Object o)
{

37
processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java
View File

@ -24,9 +24,8 @@ import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableSet;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.DruidDoublePredicate;
import org.apache.druid.query.filter.DruidFloatPredicate;
import org.apache.druid.query.filter.DruidLongPredicate;
@ -40,8 +39,10 @@ import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnProcessors;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import javax.annotation.Nullable;
import java.util.Objects;
import java.util.Set;
@ -85,10 +86,14 @@ public class DimensionPredicateFilter implements Filter
}
}
@Nullable
@Override
public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory)
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
return Filters.matchPredicate(dimension, selector, bitmapResultFactory, predicateFactory.makeStringPredicate());
if (!Filters.checkFilterTuningUseIndex(dimension, selector, filterTuning)) {
return null;
}
return Filters.makePredicateIndex(dimension, selector, predicateFactory);
}
@Override
@ -120,33 +125,11 @@ public class DimensionPredicateFilter implements Filter
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return selector.getBitmapIndex(dimension) != null;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return Filters.shouldUseBitmapIndex(this, selector, filterTuning);
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, ColumnIndexSelector indexSelector)
{
return Filters.supportsSelectivityEstimation(this, dimension, columnSelector, indexSelector);
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
{
return Filters.estimateSelectivity(
dimension,
indexSelector,
predicateFactory.makeStringPredicate()
);
}
@Override
public String toString()
{

262
processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java
View File

@ -30,8 +30,7 @@ import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExpressionType;
import org.apache.druid.math.expr.InputBindings;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.DruidDoublePredicate;
import org.apache.druid.query.filter.DruidFloatPredicate;
import org.apache.druid.query.filter.DruidLongPredicate;
@ -47,6 +46,7 @@ import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.ColumnValueSelector;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnType;
@ -54,6 +54,7 @@ import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import org.apache.druid.segment.virtual.ExpressionSelectors;
import org.apache.druid.segment.virtual.ExpressionVectorSelectors;
import javax.annotation.Nullable;
import java.util.Arrays;
import java.util.Objects;
import java.util.Set;
@ -82,51 +83,7 @@ public class ExpressionFilter implements Filter
{
final Expr theExpr = expr.get();
DruidPredicateFactory predicateFactory = new DruidPredicateFactory()
{
@Override
public Predicate<String> makeStringPredicate()
{
return Evals::asBoolean;
}
@Override
public DruidLongPredicate makeLongPredicate()
{
return Evals::asBoolean;
}
@Override
public DruidFloatPredicate makeFloatPredicate()
{
return Evals::asBoolean;
}
@Override
public DruidDoublePredicate makeDoublePredicate()
{
return Evals::asBoolean;
}
// The hashcode and equals are to make SubclassesMustOverrideEqualsAndHashCodeTest stop complaining..
// DruidPredicateFactory currently doesn't really need equals or hashcode since 'toString' method that is actually
// called when testing equality of DimensionPredicateFilter, so it's the truly required method, but that seems
// a bit strange. DimensionPredicateFilter should probably be reworked to use equals from DruidPredicateFactory
// instead of using toString.
@Override
public int hashCode()
{
return super.hashCode();
}
@Override
public boolean equals(Object obj)
{
return super.equals(obj);
}
};
DruidPredicateFactory predicateFactory = getPredicateFactory();
final ExpressionType outputType = theExpr.getOutputType(factory);
// for all vectorizable expressions, outputType will only ever be null in cases where there is absolutely no
@ -217,19 +174,22 @@ public class ExpressionFilter implements Filter
};
}
@Nullable
@Override
public boolean supportsBitmapIndex(final BitmapIndexSelector selector)
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
final Expr.BindingAnalysis details = this.bindingDetails.get();
final Expr.BindingAnalysis details = bindingDetails.get();
if (details.getRequiredBindings().isEmpty()) {
// Constant expression.
return true;
return Filters.makeNullIndex(
expr.get().eval(InputBindings.nilBindings()).asBoolean(),
selector
);
} else if (details.getRequiredBindings().size() == 1) {
// Single-column expression. We can use bitmap indexes if this column has an index and the expression can
// map over the values of the index.
final String column = Iterables.getOnlyElement(details.getRequiredBindings());
// we use a default 'all false' capabilities here because if the column has a bitmap index, but the capabilities
// are null, it means that the column is missing and should take the single valued path, while truly unknown
// things will not have a bitmap index available
@ -237,60 +197,31 @@ public class ExpressionFilter implements Filter
column,
ColumnCapabilitiesImpl.createDefault()
);
return selector.getBitmapIndex(column) != null && ExpressionSelectors.canMapOverDictionary(details, capabilities);
} else {
// Multi-column expression.
return false;
}
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return Filters.shouldUseBitmapIndex(this, selector, filterTuning);
}
@Override
public <T> T getBitmapResult(final BitmapIndexSelector selector, final BitmapResultFactory<T> bitmapResultFactory)
{
if (bindingDetails.get().getRequiredBindings().isEmpty()) {
// Constant expression.
if (expr.get().eval(InputBindings.nilBindings()).asBoolean()) {
return bitmapResultFactory.wrapAllTrue(Filters.allTrue(selector));
} else {
return bitmapResultFactory.wrapAllFalse(Filters.allFalse(selector));
if (ExpressionSelectors.canMapOverDictionary(details, capabilities)) {
if (!Filters.checkFilterTuningUseIndex(column, selector, filterTuning)) {
return null;
}
return Filters.makePredicateIndex(
column,
selector,
getBitmapPredicateFactory()
);
}
} else {
// Can assume there's only one binding, it has a bitmap index, and it's a single input mapping.
// Otherwise, supportsBitmapIndex would have returned false and the caller should not have called us.
assert supportsBitmapIndex(selector);
final String column = Iterables.getOnlyElement(bindingDetails.get().getRequiredBindings());
return Filters.matchPredicate(
column,
selector,
bitmapResultFactory,
value -> expr.get().eval(InputBindings.forFunction(identifierName -> {
// There's only one binding, and it must be the single column, so it can safely be ignored in production.
assert column.equals(identifierName);
// convert null to Empty before passing to expressions if needed.
return NullHandling.nullToEmptyIfNeeded(value);
})).asBoolean()
);
}
return null;
}
@Override
public boolean supportsSelectivityEstimation(
final ColumnSelector columnSelector,
final BitmapIndexSelector indexSelector
final ColumnIndexSelector indexSelector
)
{
return false;
}
@Override
public double estimateSelectivity(final BitmapIndexSelector indexSelector)
public double estimateSelectivity(final ColumnIndexSelector indexSelector)
{
// Selectivity estimation not supported.
throw new UnsupportedOperationException();
@ -337,4 +268,149 @@ public class ExpressionFilter implements Filter
", filterTuning=" + filterTuning +
'}';
}
/**
* {@link DruidPredicateFactory} which pipes already evaluated expression values through to {@link Evals#asBoolean},
* used for a value matcher on top of expression selectors
*/
private DruidPredicateFactory getPredicateFactory()
{
return new DruidPredicateFactory()
{
@Override
public Predicate<String> makeStringPredicate()
{
return Evals::asBoolean;
}
@Override
public DruidLongPredicate makeLongPredicate()
{
return Evals::asBoolean;
}
@Override
public DruidFloatPredicate makeFloatPredicate()
{
return Evals::asBoolean;
}
@Override
public DruidDoublePredicate makeDoublePredicate()
{
return Evals::asBoolean;
}
// The hashcode and equals are to make SubclassesMustOverrideEqualsAndHashCodeTest stop complaining..
// DruidPredicateFactory currently doesn't really need equals or hashcode since 'toString' method that is actually
// called when testing equality of DimensionPredicateFilter, so it's the truly required method, but that seems
// a bit strange. DimensionPredicateFilter should probably be reworked to use equals from DruidPredicateFactory
// instead of using toString.
@Override
public int hashCode()
{
return super.hashCode();
}
@Override
public boolean equals(Object obj)
{
return super.equals(obj);
}
};
}
/**
* {@link DruidPredicateFactory} which evaluates the expression using the value as input, used for building predicate
* indexes where the raw column values will be checked against this predicate
*/
private DruidPredicateFactory getBitmapPredicateFactory()
{
return new DruidPredicateFactory()
{
@Override
public Predicate<String> makeStringPredicate()
{
return value -> expr.get().eval(
InputBindings.forFunction(identifierName -> NullHandling.nullToEmptyIfNeeded(value))
).asBoolean();
}
@Override
public DruidLongPredicate makeLongPredicate()
{
return new DruidLongPredicate()
{
@Override
public boolean applyLong(long input)
{
return expr.get().eval(InputBindings.forFunction(identifierName -> input)).asBoolean();
}
@Override
public boolean applyNull()
{
return expr.get().eval(InputBindings.nilBindings()).asBoolean();
}
};
}
@Override
public DruidFloatPredicate makeFloatPredicate()
{
return new DruidFloatPredicate()
{
@Override
public boolean applyFloat(float input)
{
return expr.get().eval(InputBindings.forFunction(identifierName -> input)).asBoolean();
}
@Override
public boolean applyNull()
{
return expr.get().eval(InputBindings.nilBindings()).asBoolean();
}
};
}
@Override
public DruidDoublePredicate makeDoublePredicate()
{
return new DruidDoublePredicate()
{
@Override
public boolean applyDouble(double input)
{
return expr.get().eval(InputBindings.forFunction(identifierName -> input)).asBoolean();
}
@Override
public boolean applyNull()
{
return expr.get().eval(InputBindings.nilBindings()).asBoolean();
}
};
}
// The hashcode and equals are to make SubclassesMustOverrideEqualsAndHashCodeTest stop complaining..
// DruidPredicateFactory currently doesn't really need equals or hashcode since 'toString' method that is actually
// called when testing equality of DimensionPredicateFilter, so it's the truly required method, but that seems
// a bit strange. DimensionPredicateFilter should probably be reworked to use equals from DruidPredicateFactory
// instead of using toString.
@Override
public int hashCode()
{
return super.hashCode();
}
@Override
public boolean equals(Object obj)
{
return super.equals(obj);
}
};
}
}

27
processing/src/main/java/org/apache/druid/segment/filter/FalseFilter.java
View File

@ -19,8 +19,7 @@
package org.apache.druid.segment.filter;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.query.filter.vector.BooleanVectorValueMatcher;
@ -28,8 +27,11 @@ import org.apache.druid.query.filter.vector.VectorValueMatcher;
import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.column.AllFalseBitmapColumnIndex;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import javax.annotation.Nullable;
import java.util.Collections;
import java.util.Map;
import java.util.Set;
@ -47,14 +49,15 @@ public class FalseFilter implements Filter
{
}
@Nullable
@Override
public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory)
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
return bitmapResultFactory.wrapAllFalse(Filters.allFalse(selector));
return new AllFalseBitmapColumnIndex(selector);
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
public double estimateSelectivity(ColumnIndexSelector indexSelector)
{
return 0;
}
@ -72,19 +75,7 @@ public class FalseFilter implements Filter
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return true;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return true;
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, ColumnIndexSelector indexSelector)
{
return true;
}

369
processing/src/main/java/org/apache/druid/segment/filter/Filters.java
View File

@ -20,18 +20,13 @@
package org.apache.druid.segment.filter;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import it.unimi.dsi.fastutil.ints.IntIterable;
import it.unimi.dsi.fastutil.ints.IntIterator;
import it.unimi.dsi.fastutil.ints.IntList;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.DefaultBitmapResultFactory;
import org.apache.druid.query.Query;
import org.apache.druid.query.QueryContexts;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.DimFilter;
import org.apache.druid.query.filter.DruidPredicateFactory;
import org.apache.druid.query.filter.Filter;
@ -40,26 +35,25 @@ import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.segment.ColumnProcessors;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.IntIteratorUtils;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.AllFalseBitmapColumnIndex;
import org.apache.druid.segment.column.AllTrueBitmapColumnIndex;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.data.CloseableIndexed;
import org.apache.druid.segment.data.Indexed;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex;
import org.apache.druid.segment.column.DruidPredicateIndex;
import org.apache.druid.segment.filter.cnf.CNFFilterExplosionException;
import org.apache.druid.segment.filter.cnf.CalciteCnfHelper;
import org.apache.druid.segment.filter.cnf.HiveCnfHelper;
import org.apache.druid.segment.join.filter.AllNullColumnSelectorFactory;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
@ -148,192 +142,59 @@ public class Filters
);
}
public static ImmutableBitmap allFalse(final BitmapIndexSelector selector)
{
return selector.getBitmapFactory().makeEmptyImmutableBitmap();
}
public static ImmutableBitmap allTrue(final BitmapIndexSelector selector)
{
return selector.getBitmapFactory()
.complement(selector.getBitmapFactory().makeEmptyImmutableBitmap(), selector.getNumRows());
}
/**
* Transform an iterable of indexes of bitmaps to an iterable of bitmaps
*
* @param indexes indexes of bitmaps
* @param bitmapIndex an object to retrieve bitmaps using indexes
*
* @return an iterable of bitmaps
*/
public static Iterable<ImmutableBitmap> bitmapsFromIndexes(final IntIterable indexes, final BitmapIndex bitmapIndex)
{
// Do not use Iterables.transform() to avoid boxing/unboxing integers.
return new Iterable<ImmutableBitmap>()
{
@Override
public Iterator<ImmutableBitmap> iterator()
{
final IntIterator iterator = indexes.iterator();
return new Iterator<ImmutableBitmap>()
{
@Override
public boolean hasNext()
{
return iterator.hasNext();
}
@Override
public ImmutableBitmap next()
{
return bitmapIndex.getBitmap(iterator.nextInt());
}
@Override
public void remove()
{
throw new UnsupportedOperationException();
}
};
}
};
}
/**
* Return the union of bitmaps for all values matching a particular predicate.
*
* @param dimension dimension to look at
* @param selector bitmap selector
* @param bitmapResultFactory
* @param predicate predicate to use
*
* @return bitmap of matching rows
*
* @see #estimateSelectivity(String, BitmapIndexSelector, Predicate)
*/
public static <T> T matchPredicate(
final String dimension,
final BitmapIndexSelector selector,
BitmapResultFactory<T> bitmapResultFactory,
final Predicate<String> predicate
@Nullable
public static BitmapColumnIndex makePredicateIndex(
final String column,
final ColumnIndexSelector selector,
final DruidPredicateFactory predicateFactory
)
{
return bitmapResultFactory.unionDimensionValueBitmaps(matchPredicateNoUnion(dimension, selector, predicate));
}
/**
* Return an iterable of bitmaps for all values matching a particular predicate. Unioning these bitmaps
* yields the same result that {@link #matchPredicate(String, BitmapIndexSelector, BitmapResultFactory, Predicate)}
* would have returned.
*
* @param dimension dimension to look at
* @param selector bitmap selector
* @param predicate predicate to use
*
* @return iterable of bitmaps of matching rows
*/
public static Iterable<ImmutableBitmap> matchPredicateNoUnion(
final String dimension,
final BitmapIndexSelector selector,
final Predicate<String> predicate
)
{
Preconditions.checkNotNull(dimension, "dimension");
Preconditions.checkNotNull(column, "column");
Preconditions.checkNotNull(selector, "selector");
Preconditions.checkNotNull(predicate, "predicate");
// Missing dimension -> match all rows if the predicate matches null; match no rows otherwise
try (final CloseableIndexed<String> dimValues = selector.getDimensionValues(dimension)) {
if (dimValues == null || dimValues.size() == 0) {
return ImmutableList.of(predicate.apply(null) ? allTrue(selector) : allFalse(selector));
Preconditions.checkNotNull(predicateFactory, "predicateFactory");
final ColumnIndexSupplier indexSupplier = selector.getIndexSupplier(column);
if (indexSupplier != null) {
final DruidPredicateIndex predicateIndex = indexSupplier.as(DruidPredicateIndex.class);
if (predicateIndex != null) {
return predicateIndex.forPredicate(predicateFactory);
}
// Apply predicate to all dimension values and union the matching bitmaps
final BitmapIndex bitmapIndex = selector.getBitmapIndex(dimension);
return makePredicateQualifyingBitmapIterable(bitmapIndex, predicate, dimValues);
}
catch (IOException e) {
throw new UncheckedIOException(e);
// index doesn't exist
return null;
}
// missing column -> match all rows if the predicate matches null; match no rows otherwise
return predicateFactory.makeStringPredicate().apply(null)
? new AllTrueBitmapColumnIndex(selector)
: new AllFalseBitmapColumnIndex(selector);
}
/**
* Return an estimated selectivity for bitmaps of all values matching the given predicate.
*
* @param dimension dimension to look at
* @param indexSelector bitmap selector
* @param predicate predicate to use
*
* @return estimated selectivity
*
* @see #matchPredicate(String, BitmapIndexSelector, BitmapResultFactory, Predicate)
*/
public static double estimateSelectivity(
final String dimension,
final BitmapIndexSelector indexSelector,
final Predicate<String> predicate
public static BitmapColumnIndex makeNullIndex(boolean matchesNull, final ColumnIndexSelector selector)
{
return matchesNull ? new AllTrueBitmapColumnIndex(selector) : new AllFalseBitmapColumnIndex(selector);
}
public static ImmutableBitmap computeDefaultBitmapResults(Filter filter, ColumnIndexSelector selector)
{
return filter.getBitmapColumnIndex(selector).computeBitmapResult(
new DefaultBitmapResultFactory(selector.getBitmapFactory())
);
}
public static boolean supportsSelectivityEstimation(
Filter filter,
String dimension,
ColumnSelector columnSelector,
ColumnIndexSelector indexSelector
)
{
Preconditions.checkNotNull(dimension, "dimension");
Preconditions.checkNotNull(indexSelector, "selector");
Preconditions.checkNotNull(predicate, "predicate");
// Missing dimension -> match all rows if the predicate matches null; match no rows otherwise
try (final CloseableIndexed<String> dimValues = indexSelector.getDimensionValues(dimension)) {
if (dimValues == null || dimValues.size() == 0) {
return predicate.apply(null) ? 1. : 0.;
if (filter.getBitmapColumnIndex(indexSelector) != null) {
final ColumnHolder columnHolder = columnSelector.getColumnHolder(dimension);
if (columnHolder != null) {
return columnHolder.getCapabilities().hasMultipleValues().isFalse();
}
// Apply predicate to all dimension values and union the matching bitmaps
final BitmapIndex bitmapIndex = indexSelector.getBitmapIndex(dimension);
return estimateSelectivity(
bitmapIndex,
IntIteratorUtils.toIntList(
makePredicateQualifyingIndexIterable(bitmapIndex, predicate, dimValues).iterator()
),
indexSelector.getNumRows()
);
}
catch (IOException e) {
throw new UncheckedIOException(e);
}
return false;
}
/**
* Return an estimated selectivity for bitmaps for the dimension values given by dimValueIndexes.
*
* @param bitmapIndex bitmap index
* @param bitmaps bitmaps to extract, by index
* @param totalNumRows number of rows in the column associated with this bitmap index
*
* @return estimated selectivity
*/
public static double estimateSelectivity(
final BitmapIndex bitmapIndex,
final IntList bitmaps,
final long totalNumRows
)
{
long numMatchedRows = 0;
for (int i = 0; i < bitmaps.size(); i++) {
final ImmutableBitmap bitmap = bitmapIndex.getBitmap(bitmaps.getInt(i));
numMatchedRows += bitmap.size();
}
return Math.min(1., (double) numMatchedRows / totalNumRows);
}
/**
* Return an estimated selectivity for bitmaps given by an iterator.
*
* @param bitmaps iterator of bitmaps
* @param totalNumRows number of rows in the column associated with this bitmap index
*
* @return estimated selectivity
*/
public static double estimateSelectivity(
final Iterator<ImmutableBitmap> bitmaps,
final long totalNumRows
@ -345,87 +206,7 @@ public class Filters
numMatchedRows += bitmap.size();
}
return Math.min(1., (double) numMatchedRows / totalNumRows);
}
private static Iterable<ImmutableBitmap> makePredicateQualifyingBitmapIterable(
final BitmapIndex bitmapIndex,
final Predicate<String> predicate,
final Indexed<String> dimValues
)
{
return bitmapsFromIndexes(makePredicateQualifyingIndexIterable(bitmapIndex, predicate, dimValues), bitmapIndex);
}
private static IntIterable makePredicateQualifyingIndexIterable(
final BitmapIndex bitmapIndex,
final Predicate<String> predicate,
final Indexed<String> dimValues
)
{
return new IntIterable()
{
@Override
public IntIterator iterator()
{
return new IntIterator()
{
private final int bitmapIndexCardinality = bitmapIndex.getCardinality();
private int nextIndex = 0;
private int found;
{
found = findNextIndex();
}
private int findNextIndex()
{
while (nextIndex < bitmapIndexCardinality && !predicate.apply(dimValues.get(nextIndex))) {
nextIndex++;
}
if (nextIndex < bitmapIndexCardinality) {
return nextIndex++;
} else {
return -1;
}
}
@Override
public boolean hasNext()
{
return found != -1;
}
@Override
public int nextInt()
{
int foundIndex = this.found;
if (foundIndex == -1) {
throw new NoSuchElementException();
}
this.found = findNextIndex();
return foundIndex;
}
};
}
};
}
public static boolean supportsSelectivityEstimation(
Filter filter,
String dimension,
ColumnSelector columnSelector,
BitmapIndexSelector indexSelector
)
{
if (filter.supportsBitmapIndex(indexSelector)) {
final ColumnHolder columnHolder = columnSelector.getColumnHolder(dimension);
if (columnHolder != null) {
return columnHolder.getCapabilities().hasMultipleValues().isFalse();
}
}
return false;
return Math.min(1, (double) numMatchedRows / totalNumRows);
}
@Nullable
@ -463,35 +244,41 @@ public class Filters
return current;
}
/**
* This method provides a "standard" implementation of {@link Filter#shouldUseBitmapIndex(BitmapIndexSelector)} which takes
* a {@link Filter}, a {@link BitmapIndexSelector}, and {@link FilterTuning} to determine if:
* a) the filter supports bitmap indexes for all required columns
* b) the filter tuning specifies that it should use the index
* c) the cardinality of the column is above the minimum threshold and below the maximum threshold to use the index
*
* If all these things are true, {@link org.apache.druid.segment.QueryableIndexStorageAdapter} will utilize the
* indexes.
*/
public static boolean shouldUseBitmapIndex(
Filter filter,
BitmapIndexSelector indexSelector,
public static boolean checkFilterTuningUseIndex(
String columnName,
ColumnIndexSelector indexSelector,
@Nullable FilterTuning filterTuning
)
{
final FilterTuning tuning = filterTuning != null ? filterTuning : FilterTuning.createDefault(filter, indexSelector);
if (filter.supportsBitmapIndex(indexSelector) && tuning.getUseBitmapIndex()) {
return filter.getRequiredColumns().stream().allMatch(column -> {
final BitmapIndex index = indexSelector.getBitmapIndex(column);
Preconditions.checkNotNull(index, "Column does not have a bitmap index");
final int cardinality = index.getCardinality();
return cardinality >= tuning.getMinCardinalityToUseBitmapIndex()
&& cardinality <= tuning.getMaxCardinalityToUseBitmapIndex();
});
if (filterTuning != null) {
if (!filterTuning.getUseBitmapIndex()) {
return false;
}
if (filterTuning.hasValueCardinalityThreshold()) {
final ColumnIndexSupplier indexSupplier = indexSelector.getIndexSupplier(columnName);
if (indexSupplier != null) {
final DictionaryEncodedStringValueIndex valueIndex =
indexSupplier.as(DictionaryEncodedStringValueIndex.class);
if (valueIndex != null) {
final int cardinality = valueIndex.getCardinality();
Integer min = filterTuning.getMinCardinalityToUseBitmapIndex();
Integer max = filterTuning.getMaxCardinalityToUseBitmapIndex();
if (min != null && cardinality < min) {
return false;
}
if (max != null && cardinality > max) {
return false;
}
}
}
}
}
return false;
return true;
}
/**
* Create a filter representing an AND relationship across a list of filters. Deduplicates filters, flattens stacks,
* and removes null filters and literal "false" filters.

57
processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java
View File

@ -19,18 +19,17 @@
package org.apache.druid.segment.filter;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableSet;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.FilterTuning;
import org.apache.druid.query.filter.JavaScriptDimFilter;
import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.mozilla.javascript.Context;
import org.apache.druid.segment.column.BitmapColumnIndex;
import javax.annotation.Nullable;
import java.util.Objects;
import java.util.Set;
@ -51,40 +50,14 @@ public class JavaScriptFilter implements Filter
this.filterTuning = filterTuning;
}
@Nullable
@Override
public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory)
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
final Context cx = Context.enter();
try {
return Filters.matchPredicate(dimension, selector, bitmapResultFactory, makeStringPredicate(cx));
if (!Filters.checkFilterTuningUseIndex(dimension, selector, filterTuning)) {
return null;
}
finally {
Context.exit();
}
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
{
final Context cx = Context.enter();
try {
return Filters.estimateSelectivity(dimension, indexSelector, makeStringPredicate(cx));
}
finally {
Context.exit();
}
}
private Predicate<String> makeStringPredicate(final Context context)
{
return new Predicate<String>()
{
@Override
public boolean apply(String input)
{
return predicateFactory.applyInContext(context, input);
}
};
return Filters.makePredicateIndex(dimension, selector, predicateFactory);
}
@Override
@ -95,19 +68,7 @@ public class JavaScriptFilter implements Filter
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return selector.getBitmapIndex(dimension) != null;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return Filters.shouldUseBitmapIndex(this, selector, filterTuning);
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, ColumnIndexSelector indexSelector)
{
return Filters.supportsSelectivityEstimation(this, dimension, columnSelector, indexSelector);
}

109
processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java
View File

@ -19,14 +19,11 @@
package org.apache.druid.segment.filter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.FilterTuning;
import org.apache.druid.query.filter.LikeDimFilter;
@ -37,9 +34,15 @@ import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnProcessors;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.AllFalseBitmapColumnIndex;
import org.apache.druid.segment.column.AllTrueBitmapColumnIndex;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.LexicographicalRangeIndex;
import org.apache.druid.segment.column.StringValueSetIndex;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import javax.annotation.Nullable;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@ -65,15 +68,42 @@ public class LikeFilter implements Filter
}
@Override
public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory)
@Nullable
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
return bitmapResultFactory.unionDimensionValueBitmaps(getBitmapIterable(selector));
}
if (!Filters.checkFilterTuningUseIndex(dimension, selector, filterTuning)) {
return null;
}
final ColumnIndexSupplier indexSupplier = selector.getIndexSupplier(dimension);
if (indexSupplier == null) {
// Treat this as a column full of nulls
return likeMatcher.matches(null)
? new AllTrueBitmapColumnIndex(selector)
: new AllFalseBitmapColumnIndex(selector);
}
if (isSimpleEquals()) {
StringValueSetIndex valueIndex = indexSupplier.as(StringValueSetIndex.class);
if (valueIndex != null) {
return valueIndex.forValue(
NullHandling.emptyToNullIfNeeded(likeMatcher.getPrefix())
);
}
}
if (isSimplePrefix()) {
final LexicographicalRangeIndex rangeIndex = indexSupplier.as(LexicographicalRangeIndex.class);
if (rangeIndex != null) {
final String lower = NullHandling.nullToEmptyIfNeeded(likeMatcher.getPrefix());
final String upper = NullHandling.nullToEmptyIfNeeded(likeMatcher.getPrefix()) + Character.MAX_VALUE;
return rangeIndex.forRange(lower, false, upper, false, likeMatcher::matchesSuffixOnly);
}
}
@Override
public double estimateSelectivity(BitmapIndexSelector selector)
{
return Filters.estimateSelectivity(getBitmapIterable(selector).iterator(), selector.getNumRows());
// fallback to predicate index
return Filters.makePredicateIndex(
dimension,
selector,
likeMatcher.predicateFactory(extractionFn)
);
}
@Override
@ -132,64 +162,11 @@ public class LikeFilter implements Filter
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return selector.getBitmapIndex(dimension) != null;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return Filters.shouldUseBitmapIndex(this, selector, filterTuning);
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, ColumnIndexSelector indexSelector)
{
return Filters.supportsSelectivityEstimation(this, dimension, columnSelector, indexSelector);
}
private Iterable<ImmutableBitmap> getBitmapIterable(final BitmapIndexSelector selector)
{
if (isSimpleEquals()) {
// Verify that dimension equals prefix.
return ImmutableList.of(
selector.getBitmapIndex(
dimension,
NullHandling.emptyToNullIfNeeded(likeMatcher.getPrefix())
)
);
} else if (isSimplePrefix()) {
// Verify that dimension startsWith prefix, and is accepted by likeMatcher.matchesSuffixOnly.
final BitmapIndex bitmapIndex = selector.getBitmapIndex(dimension);
if (bitmapIndex == null) {
// Treat this as a column full of nulls
return ImmutableList.of(likeMatcher.matches(null) ? Filters.allTrue(selector) : Filters.allFalse(selector));
}
final String lower = NullHandling.nullToEmptyIfNeeded(likeMatcher.getPrefix());
final String upper = NullHandling.nullToEmptyIfNeeded(likeMatcher.getPrefix()) + Character.MAX_VALUE;
// Union bitmaps for all matching dimension values in range.
// Use lazy iterator to allow unioning bitmaps one by one and avoid materializing all of them at once.
return bitmapIndex.getBitmapsInRange(
lower,
false,
upper,
false,
(value) -> likeMatcher.matchesSuffixOnly(value)
);
} else {
// fallback
return Filters.matchPredicateNoUnion(
dimension,
selector,
likeMatcher.predicateFactory(extractionFn).makeStringPredicate()
);
}
}
/**
* Returns true if this filter is a simple equals filter: dimension = 'value' with no extractionFn.
*/

59
processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java
View File

@ -21,7 +21,7 @@ package org.apache.druid.segment.filter;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.query.filter.vector.BaseVectorValueMatcher;
@ -32,8 +32,11 @@ import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnIndexCapabilities;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import javax.annotation.Nullable;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@ -49,13 +52,37 @@ public class NotFilter implements Filter
this.baseFilter = baseFilter;
}
@Nullable
@Override
public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory)
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
return bitmapResultFactory.complement(
baseFilter.getBitmapResult(selector, bitmapResultFactory),
selector.getNumRows()
);
final BitmapColumnIndex baseIndex = baseFilter.getBitmapColumnIndex(selector);
if (baseIndex != null && baseIndex.getIndexCapabilities().isInvertible()) {
return new BitmapColumnIndex()
{
@Override
public ColumnIndexCapabilities getIndexCapabilities()
{
return baseIndex.getIndexCapabilities();
}
@Override
public double estimateSelectivity(int totalRows)
{
return 1. - baseFilter.estimateSelectivity(selector);
}
@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory)
{
return bitmapResultFactory.complement(
baseIndex.computeBitmapResult(bitmapResultFactory),
selector.getNumRows()
);
}
};
}
return null;
}
@Override
@ -126,29 +153,11 @@ public class NotFilter implements Filter
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return baseFilter.supportsBitmapIndex(selector);
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return baseFilter.shouldUseBitmapIndex(selector);
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, ColumnIndexSelector indexSelector)
{
return baseFilter.supportsSelectivityEstimation(columnSelector, indexSelector);
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
{
return 1. - baseFilter.estimateSelectivity(indexSelector);
}
@Override
public String toString()
{

73
processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java
View File

@ -22,11 +22,13 @@ package org.apache.druid.segment.filter;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterables;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.DefaultBitmapResultFactory;
import org.apache.druid.query.filter.BooleanFilter;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.RowOffsetMatcherFactory;
import org.apache.druid.query.filter.ValueMatcher;
@ -37,8 +39,12 @@ import org.apache.druid.query.filter.vector.VectorValueMatcher;
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnIndexCapabilities;
import org.apache.druid.segment.column.SimpleColumnIndexCapabilities;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
@ -64,19 +70,54 @@ public class OrFilter implements BooleanFilter
this(new LinkedHashSet<>(filters));
}
@Nullable
@Override
public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory)
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
if (filters.size() == 1) {
return Iterables.getOnlyElement(filters).getBitmapResult(selector, bitmapResultFactory);
return Iterables.getOnlyElement(filters).getBitmapColumnIndex(selector);
}
List<T> bitmapResults = new ArrayList<>();
List<BitmapColumnIndex> bitmapColumnIndices = new ArrayList<>(filters.size());
ColumnIndexCapabilities merged = new SimpleColumnIndexCapabilities(true, true);
for (Filter filter : filters) {
bitmapResults.add(filter.getBitmapResult(selector, bitmapResultFactory));
BitmapColumnIndex index = filter.getBitmapColumnIndex(selector);
if (index == null) {
// all or nothing
return null;
}
merged = merged.merge(index.getIndexCapabilities());
bitmapColumnIndices.add(index);
}
return bitmapResultFactory.union(bitmapResults);
final ColumnIndexCapabilities finalMerged = merged;
return new BitmapColumnIndex()
{
@Override
public ColumnIndexCapabilities getIndexCapabilities()
{
return finalMerged;
}
@Override
public double estimateSelectivity(int totalRows)
{
// Estimate selectivity with attribute value independence assumption
double selectivity = 0;
for (final Filter filter : filters) {
selectivity += filter.estimateSelectivity(selector);
}
return Math.min(selectivity, 1.);
}
@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory)
{
return bitmapResultFactory.union(
() -> bitmapColumnIndices.stream().map(x -> x.computeBitmapResult(bitmapResultFactory)).iterator()
);
}
};
}
@Override
@ -111,17 +152,20 @@ public class OrFilter implements BooleanFilter
@Override
public ValueMatcher makeMatcher(
BitmapIndexSelector selector,
ColumnIndexSelector selector,
ColumnSelectorFactory columnSelectorFactory,
RowOffsetMatcherFactory rowOffsetMatcherFactory
)
{
final List<ValueMatcher> matchers = new ArrayList<>();
final List<ImmutableBitmap> bitmaps = new ArrayList<>();
final BitmapFactory bitmapFactory = selector.getBitmapFactory();
final DefaultBitmapResultFactory resultFactory = new DefaultBitmapResultFactory(bitmapFactory);
for (Filter filter : filters) {
if (filter.supportsBitmapIndex(selector)) {
bitmaps.add(filter.getBitmapIndex(selector));
final BitmapColumnIndex columnIndex = filter.getBitmapColumnIndex(selector);
if (columnIndex != null && columnIndex.getIndexCapabilities().isExact()) {
bitmaps.add(columnIndex.computeBitmapResult(resultFactory));
} else {
ValueMatcher matcher = filter.makeMatcher(columnSelectorFactory);
matchers.add(matcher);
@ -143,17 +187,6 @@ public class OrFilter implements BooleanFilter
return filters;
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
{
// Estimate selectivity with attribute value independence assumption
double selectivity = 0;
for (final Filter filter : filters) {
selectivity += filter.estimateSelectivity(indexSelector);
}
return Math.min(selectivity, 1.);
}
@Override
public String toString()
{

44
processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java
View File

@ -20,10 +20,10 @@
package org.apache.druid.segment.filter;
import com.google.common.collect.ImmutableSet;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.FilterTuning;
import org.apache.druid.query.filter.ValueMatcher;
@ -33,6 +33,9 @@ import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnProcessors;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.StringValueSetIndex;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import javax.annotation.Nullable;
@ -74,9 +77,22 @@ public class SelectorFilter implements Filter
}
@Override
public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory)
@Nullable
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
return bitmapResultFactory.wrapDimensionValue(selector.getBitmapIndex(dimension, value));
if (!Filters.checkFilterTuningUseIndex(dimension, selector, filterTuning)) {
return null;
}
final ColumnIndexSupplier indexSupplier = selector.getIndexSupplier(dimension);
if (indexSupplier == null) {
return Filters.makeNullIndex(NullHandling.isNullOrEquivalent(value), selector);
}
final StringValueSetIndex valueSetIndex = indexSupplier.as(StringValueSetIndex.class);
if (valueSetIndex == null) {
// column exists, but has no index
return null;
}
return valueSetIndex.forValue(value);
}
@Override
@ -96,29 +112,11 @@ public class SelectorFilter implements Filter
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return selector.getBitmapIndex(dimension) != null;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return Filters.shouldUseBitmapIndex(this, selector, filterTuning);
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, ColumnIndexSelector indexSelector)
{
return Filters.supportsSelectivityEstimation(this, dimension, columnSelector, indexSelector);
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
{
return (double) indexSelector.getBitmapIndex(dimension, value).size() / indexSelector.getNumRows();
}
@Override
public boolean canVectorizeMatcher(ColumnInspector inspector)
{

60
processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java
View File

@ -26,7 +26,7 @@ import com.google.common.collect.ImmutableSet;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.collections.spatial.search.Bound;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.DruidDoublePredicate;
import org.apache.druid.query.filter.DruidFloatPredicate;
import org.apache.druid.query.filter.DruidLongPredicate;
@ -36,8 +36,15 @@ import org.apache.druid.query.filter.FilterTuning;
import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.column.AllFalseBitmapColumnIndex;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnIndexCapabilities;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.SimpleColumnIndexCapabilities;
import org.apache.druid.segment.column.SpatialIndex;
import org.apache.druid.segment.incremental.SpatialDimensionRowTransformer;
import javax.annotation.Nullable;
import java.util.Objects;
import java.util.Set;
@ -60,11 +67,40 @@ public class SpatialFilter implements Filter
this.filterTuning = filterTuning;
}
@Nullable
@Override
public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory)
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
Iterable<ImmutableBitmap> search = selector.getSpatialIndex(dimension).search(bound);
return bitmapResultFactory.unionDimensionValueBitmaps(search);
if (!Filters.checkFilterTuningUseIndex(dimension, selector, filterTuning)) {
return null;
}
final ColumnIndexSupplier indexSupplier = selector.getIndexSupplier(dimension);
final SpatialIndex spatialIndex = indexSupplier == null ? null : indexSupplier.as(SpatialIndex.class);
if (spatialIndex == null) {
return new AllFalseBitmapColumnIndex(selector);
}
return new BitmapColumnIndex()
{
@Override
public ColumnIndexCapabilities getIndexCapabilities()
{
return new SimpleColumnIndexCapabilities(true, true);
}
@Override
public double estimateSelectivity(int totalRows)
{
// selectivity estimation for multi-value columns is not implemented yet.
throw new UnsupportedOperationException();
}
@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory)
{
Iterable<ImmutableBitmap> search = spatialIndex.getRTree().search(bound);
return bitmapResultFactory.unionDimensionValueBitmaps(search);
}
};
}
@Override
@ -79,19 +115,7 @@ public class SpatialFilter implements Filter
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return selector.getBitmapIndex(dimension) != null;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return Filters.shouldUseBitmapIndex(this, selector, filterTuning);
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, ColumnIndexSelector indexSelector)
{
return false;
}
@ -103,7 +127,7 @@ public class SpatialFilter implements Filter
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
public double estimateSelectivity(ColumnIndexSelector indexSelector)
{
// selectivity estimation for multi-value columns is not implemented yet.
throw new UnsupportedOperationException();

27
processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java
View File

@ -19,8 +19,7 @@
package org.apache.druid.segment.filter;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.query.filter.vector.BooleanVectorValueMatcher;
@ -28,8 +27,11 @@ import org.apache.druid.query.filter.vector.VectorValueMatcher;
import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.column.AllTrueBitmapColumnIndex;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import javax.annotation.Nullable;
import java.util.Collections;
import java.util.Map;
import java.util.Set;
@ -50,10 +52,11 @@ public class TrueFilter implements Filter
{
}
@Nullable
@Override
public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory)
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
return bitmapResultFactory.wrapAllTrue(Filters.allTrue(selector));
return new AllTrueBitmapColumnIndex(selector);
}
@Override
@ -69,19 +72,7 @@ public class TrueFilter implements Filter
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return true;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return true;
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, ColumnIndexSelector indexSelector)
{
return true;
}
@ -111,7 +102,7 @@ public class TrueFilter implements Filter
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
public double estimateSelectivity(ColumnIndexSelector indexSelector)
{
return 1;
}

30
processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java
View File

@ -344,25 +344,33 @@ public class DictionaryEncodedColumnPartSerde implements ColumnPartSerde
.setHasNulls(firstDictionaryEntry == null)
.setDictionaryEncodedColumnSupplier(dictionaryEncodedColumnSupplier);
GenericIndexed<ImmutableBitmap> rBitmaps = null;
ImmutableRTree rSpatialIndex = null;
if (!Feature.NO_BITMAP_INDEX.isSet(rFlags)) {
GenericIndexed<ImmutableBitmap> rBitmaps = GenericIndexed.read(
rBitmaps = GenericIndexed.read(
buffer,
bitmapSerdeFactory.getObjectStrategy(),
builder.getFileMapper()
);
builder.setBitmapIndex(
new StringBitmapIndexColumnPartSupplier(
bitmapSerdeFactory.getBitmapFactory(),
rBitmaps,
rDictionary
)
);
}
if (buffer.hasRemaining()) {
ImmutableRTree rSpatialIndex =
new ImmutableRTreeObjectStrategy(bitmapSerdeFactory.getBitmapFactory()).fromByteBufferWithSize(buffer);
builder.setSpatialIndex(new SpatialIndexColumnPartSupplier(rSpatialIndex));
rSpatialIndex = new ImmutableRTreeObjectStrategy(
bitmapSerdeFactory.getBitmapFactory()
).fromByteBufferWithSize(buffer);
}
if (rBitmaps != null || rSpatialIndex != null) {
builder.setIndexSupplier(
new DictionaryEncodedStringIndexSupplier(
bitmapSerdeFactory.getBitmapFactory(),
rDictionary,
rBitmaps,
rSpatialIndex
),
rBitmaps != null,
rSpatialIndex != null
);
}
}

513
processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedStringIndexSupplier.java Normal file
View File

@ -0,0 +1,513 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment.serde;
import com.google.common.base.Predicate;
import it.unimi.dsi.fastutil.ints.IntIntImmutablePair;
import it.unimi.dsi.fastutil.ints.IntIntPair;
import it.unimi.dsi.fastutil.ints.IntIterator;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.collections.spatial.ImmutableRTree;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.DruidPredicateFactory;
import org.apache.druid.segment.IntListUtils;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnIndexCapabilities;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex;
import org.apache.druid.segment.column.DruidPredicateIndex;
import org.apache.druid.segment.column.LexicographicalRangeIndex;
import org.apache.druid.segment.column.SimpleColumnIndexCapabilities;
import org.apache.druid.segment.column.SpatialIndex;
import org.apache.druid.segment.column.StringValueSetIndex;
import org.apache.druid.segment.data.GenericIndexed;
import org.apache.druid.segment.filter.Filters;
import javax.annotation.Nullable;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Set;
public class DictionaryEncodedStringIndexSupplier implements ColumnIndexSupplier
{
public static final ColumnIndexCapabilities CAPABILITIES = new SimpleColumnIndexCapabilities(true, true);
private final BitmapFactory bitmapFactory;
private final GenericIndexed<String> dictionary;
@Nullable
private final GenericIndexed<ImmutableBitmap> bitmaps;
@Nullable
private final ImmutableRTree indexedTree;
public DictionaryEncodedStringIndexSupplier(
BitmapFactory bitmapFactory,
GenericIndexed<String> dictionary,
GenericIndexed<ImmutableBitmap> bitmaps,
ImmutableRTree indexedTree
)
{
this.bitmapFactory = bitmapFactory;
this.bitmaps = bitmaps;
this.dictionary = dictionary;
this.indexedTree = indexedTree;
}
@Nullable
@Override
public <T> T as(Class<T> clazz)
{
if (clazz.equals(StringValueSetIndex.class)) {
return (T) new GenericIndexedDictionaryEncodedStringValueSetIndex(bitmapFactory, dictionary, bitmaps);
} else if (clazz.equals(DruidPredicateIndex.class)) {
return (T) new GenericIndexedDictionaryEncodedStringDruidPredicateIndex(bitmapFactory, dictionary, bitmaps);
} else if (clazz.equals(LexicographicalRangeIndex.class)) {
return (T) new GenericIndexedDictionaryEncodedColumnLexicographicalRangeIndex(bitmapFactory, dictionary, bitmaps);
} else if (clazz.equals(DictionaryEncodedStringValueIndex.class)) {
return (T) new GenericIndexedDictionaryEncodedStringValueIndex(bitmapFactory, dictionary, bitmaps);
} else if (clazz.equals(SpatialIndex.class)) {
return (T) (SpatialIndex) () -> indexedTree;
}
return null;
}
private abstract static class DictionaryEncodedStringBitmapColumnIndex implements BitmapColumnIndex
{
@Override
public ColumnIndexCapabilities getIndexCapabilities()
{
return CAPABILITIES;
}
}
private abstract static class BaseGenericIndexedDictionaryEncodedStringIndex
{
protected final BitmapFactory bitmapFactory;
protected final GenericIndexed<String> dictionary;
protected final GenericIndexed<ImmutableBitmap> bitmaps;
protected BaseGenericIndexedDictionaryEncodedStringIndex(
BitmapFactory bitmapFactory,
GenericIndexed<String> dictionary,
GenericIndexed<ImmutableBitmap> bitmaps
)
{
this.bitmapFactory = bitmapFactory;
this.dictionary = dictionary;
this.bitmaps = bitmaps;
}
public ImmutableBitmap getBitmap(int idx)
{
if (idx < 0) {
return bitmapFactory.makeEmptyImmutableBitmap();
}
final ImmutableBitmap bitmap = bitmaps.get(idx);
return bitmap == null ? bitmapFactory.makeEmptyImmutableBitmap() : bitmap;
}
}
public static final class GenericIndexedDictionaryEncodedStringValueIndex
extends BaseGenericIndexedDictionaryEncodedStringIndex implements DictionaryEncodedStringValueIndex
{
public GenericIndexedDictionaryEncodedStringValueIndex(
BitmapFactory bitmapFactory,
GenericIndexed<String> dictionary,
GenericIndexed<ImmutableBitmap> bitmaps
)
{
super(bitmapFactory, dictionary, bitmaps);
}
@Override
public boolean hasNulls()
{
return dictionary.indexOf(null) >= 0;
}
@Override
public int getCardinality()
{
return dictionary.size();
}
@Nullable
@Override
public String getValue(int index)
{
return dictionary.get(index);
}
@Override
public int getIndex(@Nullable String value)
{
return dictionary.indexOf(value);
}
}
public static final class GenericIndexedDictionaryEncodedStringValueSetIndex
extends BaseGenericIndexedDictionaryEncodedStringIndex implements StringValueSetIndex
{
public GenericIndexedDictionaryEncodedStringValueSetIndex(
BitmapFactory bitmapFactory,
GenericIndexed<String> dictionary,
GenericIndexed<ImmutableBitmap> bitmaps
)
{
super(bitmapFactory, dictionary, bitmaps);
}
@Override
public BitmapColumnIndex forValue(@Nullable String value)
{
return new DictionaryEncodedStringBitmapColumnIndex()
{
@Override
public double estimateSelectivity(int totalRows)
{
return Math.min(1, (double) getBitmapForValue().size() / totalRows);
}
@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory)
{
return bitmapResultFactory.wrapDimensionValue(getBitmapForValue());
}
private ImmutableBitmap getBitmapForValue()
{
final int idx = dictionary.indexOf(value);
return getBitmap(idx);
}
};
}
@Override
public BitmapColumnIndex forValues(Set<String> values)
{
return new DictionaryEncodedStringBitmapColumnIndex()
{
@Override
public double estimateSelectivity(int totalRows)
{
return Filters.estimateSelectivity(getBitmapsIterable().iterator(), totalRows);
}
@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory)
{
return bitmapResultFactory.unionDimensionValueBitmaps(getBitmapsIterable());
}
private Iterable<ImmutableBitmap> getBitmapsIterable()
{
return () -> new Iterator<ImmutableBitmap>()
{
final Iterator<String> iterator = values.iterator();
int next = -1;
@Override
public boolean hasNext()
{
if (next < 0) {
findNext();
}
return next >= 0;
}
@Override
public ImmutableBitmap next()
{
if (next < 0) {
findNext();
if (next < 0) {
throw new NoSuchElementException();
}
}
final int swap = next;
next = -1;
return getBitmap(swap);
}
private void findNext()
{
while (next < 0 && iterator.hasNext()) {
String nextValue = iterator.next();
next = dictionary.indexOf(nextValue);
}
}
};
}
};
}
}
public static final class GenericIndexedDictionaryEncodedStringDruidPredicateIndex
extends BaseGenericIndexedDictionaryEncodedStringIndex implements DruidPredicateIndex
{
public GenericIndexedDictionaryEncodedStringDruidPredicateIndex(
BitmapFactory bitmapFactory,
GenericIndexed<String> dictionary,
GenericIndexed<ImmutableBitmap> bitmaps
)
{
super(bitmapFactory, dictionary, bitmaps);
}
@Override
public BitmapColumnIndex forPredicate(DruidPredicateFactory matcherFactory)
{
return new DictionaryEncodedStringBitmapColumnIndex()
{
@Override
public double estimateSelectivity(int totalRows)
{
return Filters.estimateSelectivity(
getBitmapIterable().iterator(),
totalRows
);
}
@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory)
{
return bitmapResultFactory.unionDimensionValueBitmaps(getBitmapIterable());
}
private Iterable<ImmutableBitmap> getBitmapIterable()
{
return () -> new Iterator<ImmutableBitmap>()
{
final Predicate<String> stringPredicate = matcherFactory.makeStringPredicate();
final Iterator<String> iterator = dictionary.iterator();
@Nullable
String next = null;
boolean nextSet = false;
@Override
public boolean hasNext()
{
if (!nextSet) {
findNext();
}
return nextSet;
}
@Override
public ImmutableBitmap next()
{
if (!nextSet) {
findNext();
if (!nextSet) {
throw new NoSuchElementException();
}
}
nextSet = false;
final int idx = dictionary.indexOf(next);
if (idx < 0) {
return bitmapFactory.makeEmptyImmutableBitmap();
}
final ImmutableBitmap bitmap = bitmaps.get(idx);
return bitmap == null ? bitmapFactory.makeEmptyImmutableBitmap() : bitmap;
}
private void findNext()
{
while (!nextSet && iterator.hasNext()) {
String nextValue = iterator.next();
nextSet = stringPredicate.apply(nextValue);
if (nextSet) {
next = nextValue;
}
}
}
};
}
};
}
}
public static final class GenericIndexedDictionaryEncodedColumnLexicographicalRangeIndex
extends BaseGenericIndexedDictionaryEncodedStringIndex implements LexicographicalRangeIndex
{
public GenericIndexedDictionaryEncodedColumnLexicographicalRangeIndex(
BitmapFactory bitmapFactory,
GenericIndexed<String> dictionary,
GenericIndexed<ImmutableBitmap> bitmaps
)
{
super(bitmapFactory, dictionary, bitmaps);
}
@Override
public BitmapColumnIndex forRange(
@Nullable String startValue,
boolean startStrict,
@Nullable String endValue,
boolean endStrict
)
{
return new DictionaryEncodedStringBitmapColumnIndex()
{
@Override
public double estimateSelectivity(int totalRows)
{
return Filters.estimateSelectivity(getBitmapIterable().iterator(), totalRows);
}
@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory)
{
return bitmapResultFactory.unionDimensionValueBitmaps(getBitmapIterable());
}
private Iterable<ImmutableBitmap> getBitmapIterable()
{
final IntIntPair range = getRange(startValue, startStrict, endValue, endStrict);
final int start = range.leftInt(), end = range.rightInt();
return () -> new Iterator<ImmutableBitmap>()
{
final IntIterator rangeIterator = IntListUtils.fromTo(start, end).iterator();
@Override
public boolean hasNext()
{
return rangeIterator.hasNext();
}
@Override
public ImmutableBitmap next()
{
return getBitmap(rangeIterator.nextInt());
}
};
}
};
}
@Override
public BitmapColumnIndex forRange(
@Nullable String startValue,
boolean startStrict,
@Nullable String endValue,
boolean endStrict,
Predicate<String> matcher
)
{
return new DictionaryEncodedStringBitmapColumnIndex()
{
@Override
public double estimateSelectivity(int totalRows)
{
return Filters.estimateSelectivity(getBitmapIterable().iterator(), totalRows);
}
@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory)
{
return bitmapResultFactory.unionDimensionValueBitmaps(getBitmapIterable());
}
private Iterable<ImmutableBitmap> getBitmapIterable()
{
final IntIntPair range = getRange(startValue, startStrict, endValue, endStrict);
final int start = range.leftInt(), end = range.rightInt();
return () -> new Iterator<ImmutableBitmap>()
{
int currIndex = start;
int found;
{
found = findNext();
}
private int findNext()
{
while (currIndex < end && !matcher.apply(dictionary.get(currIndex))) {
currIndex++;
}
if (currIndex < end) {
return currIndex++;
} else {
return -1;
}
}
@Override
public boolean hasNext()
{
return found != -1;
}
@Override
public ImmutableBitmap next()
{
int cur = found;
if (cur == -1) {
throw new NoSuchElementException();
}
found = findNext();
return getBitmap(cur);
}
};
}
};
}
private IntIntPair getRange(@Nullable String startValue, boolean startStrict, @Nullable String endValue, boolean endStrict)
{
int startIndex, endIndex;
if (startValue == null) {
startIndex = 0;
} else {
final int found = dictionary.indexOf(NullHandling.emptyToNullIfNeeded(startValue));
if (found >= 0) {
startIndex = startStrict ? found + 1 : found;
} else {
startIndex = -(found + 1);
}
}
if (endValue == null) {
endIndex = dictionary.size();
} else {
final int found = dictionary.indexOf(NullHandling.emptyToNullIfNeeded(endValue));
if (found >= 0) {
endIndex = endStrict ? found : found + 1;
} else {
endIndex = -(found + 1);
}
}
endIndex = Math.max(startIndex, endIndex);
return new IntIntImmutablePair(startIndex, endIndex);
}
}
}

30
processing/src/main/java/org/apache/druid/segment/serde/SpatialIndexColumnPartSupplier.java → processing/src/main/java/org/apache/druid/segment/serde/NoIndexesColumnIndexSupplier.java
View File

@ -19,33 +19,27 @@
package org.apache.druid.segment.serde;
import com.google.common.base.Supplier;
import org.apache.druid.collections.spatial.ImmutableRTree;
import org.apache.druid.segment.column.SpatialIndex;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import javax.annotation.Nullable;
/**
* Default implementation of {@link ColumnIndexSupplier} for columns which do not
* have any indexes.
*/
public class SpatialIndexColumnPartSupplier implements Supplier<SpatialIndex>
public class NoIndexesColumnIndexSupplier implements ColumnIndexSupplier
{
private final ImmutableRTree indexedTree;
private static final NoIndexesColumnIndexSupplier INSTANCE = new NoIndexesColumnIndexSupplier();
public SpatialIndexColumnPartSupplier(
ImmutableRTree indexedTree
)
public static NoIndexesColumnIndexSupplier getInstance()
{
this.indexedTree = indexedTree;
return INSTANCE;
}
@Nullable
@Override
public SpatialIndex get()
public <T> T as(Class<T> clazz)
{
return new SpatialIndex()
{
@Override
public ImmutableRTree getRTree()
{
return indexedTree;
}
};
return null;
}
}

35
processing/src/main/java/org/apache/druid/segment/serde/NullColumnPartSerde.java
View File

@ -25,10 +25,7 @@ import com.google.common.base.Suppliers;
import org.apache.druid.java.util.common.io.smoosh.FileSmoosher;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.segment.DimensionSelector;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.BitmapIndexes;
import org.apache.druid.segment.column.DictionaryEncodedColumn;
import org.apache.druid.segment.data.BitmapSerdeFactory;
import org.apache.druid.segment.data.IndexedInts;
import org.apache.druid.segment.data.ReadableOffset;
import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector;
@ -47,6 +44,7 @@ import java.util.Objects;
*/
public class NullColumnPartSerde implements ColumnPartSerde
{
private static final Serializer NOOP_SERIALIZER = new Serializer()
{
@Override
@ -62,20 +60,15 @@ public class NullColumnPartSerde implements ColumnPartSerde
};
private final int numRows;
private final BitmapSerdeFactory bitmapSerdeFactory;
private final NullDictionaryEncodedColumn nullDictionaryEncodedColumn;
private final BitmapIndex bitmapIndex;
@JsonCreator
public NullColumnPartSerde(
@JsonProperty("numRows") int numRows,
@JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory
@JsonProperty("numRows") int numRows
)
{
this.numRows = numRows;
this.bitmapSerdeFactory = bitmapSerdeFactory;
this.nullDictionaryEncodedColumn = new NullDictionaryEncodedColumn();
this.bitmapIndex = BitmapIndexes.forNilColumn(() -> numRows, bitmapSerdeFactory.getBitmapFactory());
}
@JsonProperty
@ -84,12 +77,6 @@ public class NullColumnPartSerde implements ColumnPartSerde
return numRows;
}
@JsonProperty
public BitmapSerdeFactory getBitmapSerdeFactory()
{
return bitmapSerdeFactory;
}
@Nullable
@Override
public Serializer getSerializer()
@ -101,12 +88,13 @@ public class NullColumnPartSerde implements ColumnPartSerde
public Deserializer getDeserializer()
{
return (buffer, builder, columnConfig) -> {
builder
.setHasMultipleValues(false)
.setHasNulls(true)
.setFilterable(true)
.setBitmapIndex(Suppliers.ofInstance(bitmapIndex));
builder.setDictionaryEncodedColumnSupplier(Suppliers.ofInstance(nullDictionaryEncodedColumn));
builder.setHasMultipleValues(false)
.setHasNulls(true)
.setFilterable(true)
// this is a bit sneaky, we set supplier to null here to act like a null column instead of a column
// without any indexes, which is the default state
.setIndexSupplier(null, true, false)
.setDictionaryEncodedColumnSupplier(Suppliers.ofInstance(nullDictionaryEncodedColumn));
};
}
@ -120,14 +108,13 @@ public class NullColumnPartSerde implements ColumnPartSerde
return false;
}
NullColumnPartSerde partSerde = (NullColumnPartSerde) o;
return numRows == partSerde.numRows
&& bitmapSerdeFactory.equals(partSerde.bitmapSerdeFactory);
return numRows == partSerde.numRows;
}
@Override
public int hashCode()
{
return Objects.hash(numRows, bitmapSerdeFactory);
return Objects.hash(numRows);
}
private final class NullDictionaryEncodedColumn implements DictionaryEncodedColumn<String>

265
processing/src/main/java/org/apache/druid/segment/serde/StringBitmapIndexColumnPartSupplier.java
View File

@ -1,265 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment.serde;
import com.google.common.base.Supplier;
import it.unimi.dsi.fastutil.ints.IntIntImmutablePair;
import it.unimi.dsi.fastutil.ints.IntIntPair;
import it.unimi.dsi.fastutil.ints.IntIterator;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.segment.IntListUtils;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.data.GenericIndexed;
import javax.annotation.Nullable;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.function.Predicate;
/**
* Provides {@link BitmapIndex} for some dictionary encoded column, where the dictionary and bitmaps are stored in some
* {@link GenericIndexed}.
*/
public class StringBitmapIndexColumnPartSupplier implements Supplier<BitmapIndex>
{
private final BitmapFactory bitmapFactory;
private final GenericIndexed<ImmutableBitmap> bitmaps;
private final GenericIndexed<String> dictionary;
public StringBitmapIndexColumnPartSupplier(
BitmapFactory bitmapFactory,
GenericIndexed<ImmutableBitmap> bitmaps,
GenericIndexed<String> dictionary
)
{
this.bitmapFactory = bitmapFactory;
this.bitmaps = bitmaps;
this.dictionary = dictionary;
}
@Override
public BitmapIndex get()
{
return new BitmapIndex()
{
@Override
public int getCardinality()
{
return dictionary.size();
}
@Override
public String getValue(int index)
{
return dictionary.get(index);
}
@Override
public boolean hasNulls()
{
return dictionary.indexOf(null) >= 0;
}
@Override
public BitmapFactory getBitmapFactory()
{
return bitmapFactory;
}
@Override
public int getIndex(@Nullable String value)
{
return dictionary.indexOf(value);
}
@Override
public ImmutableBitmap getBitmap(int idx)
{
if (idx < 0) {
return bitmapFactory.makeEmptyImmutableBitmap();
}
final ImmutableBitmap bitmap = bitmaps.get(idx);
return bitmap == null ? bitmapFactory.makeEmptyImmutableBitmap() : bitmap;
}
@Override
public ImmutableBitmap getBitmapForValue(@Nullable String value)
{
final int idx = dictionary.indexOf(value);
return getBitmap(idx);
}
@Override
public Iterable<ImmutableBitmap> getBitmapsInRange(
@Nullable String startValue,
boolean startStrict,
@Nullable String endValue,
boolean endStrict
)
{
final IntIntPair range = getRange(startValue, startStrict, endValue, endStrict);
final int start = range.leftInt(), end = range.rightInt();
return () -> new Iterator<ImmutableBitmap>()
{
final IntIterator rangeIterator = IntListUtils.fromTo(start, end).iterator();
@Override
public boolean hasNext()
{
return rangeIterator.hasNext();
}
@Override
public ImmutableBitmap next()
{
return getBitmap(rangeIterator.nextInt());
}
};
}
@Override
public Iterable<ImmutableBitmap> getBitmapsInRange(
@Nullable String startValue,
boolean startStrict,
@Nullable String endValue,
boolean endStrict,
Predicate<String> indexMatcher
)
{
final IntIntPair range = getRange(startValue, startStrict, endValue, endStrict);
final int start = range.leftInt(), end = range.rightInt();
return () -> new Iterator<ImmutableBitmap>()
{
int currIndex = start;
int found;
{
found = findNext();
}
private int findNext()
{
while (currIndex < end && !indexMatcher.test(dictionary.get(currIndex))) {
currIndex++;
}
if (currIndex < end) {
return currIndex++;
} else {
return -1;
}
}
@Override
public boolean hasNext()
{
return found != -1;
}
@Override
public ImmutableBitmap next()
{
int cur = found;
if (cur == -1) {
throw new NoSuchElementException();
}
found = findNext();
return getBitmap(cur);
}
};
}
@Override
public Iterable<ImmutableBitmap> getBitmapsForValues(Set<String> values)
{
return () -> new Iterator<ImmutableBitmap>()
{
final Iterator<String> iterator = values.iterator();
int next = -1;
@Override
public boolean hasNext()
{
if (next < 0) {
findNext();
}
return next >= 0;
}
@Override
public ImmutableBitmap next()
{
if (next < 0) {
findNext();
if (next < 0) {
throw new NoSuchElementException();
}
}
final int swap = next;
next = -1;
return getBitmap(swap);
}
private void findNext()
{
while (next < 0 && iterator.hasNext()) {
String nextValue = iterator.next();
next = dictionary.indexOf(nextValue);
}
}
};
}
private IntIntPair getRange(@Nullable String startValue, boolean startStrict, @Nullable String endValue, boolean endStrict)
{
int startIndex, endIndex;
if (startValue == null) {
startIndex = 0;
} else {
final int found = dictionary.indexOf(NullHandling.emptyToNullIfNeeded(startValue));
if (found >= 0) {
startIndex = startStrict ? found + 1 : found;
} else {
startIndex = -(found + 1);
}
}
if (endValue == null) {
endIndex = dictionary.size();
} else {
final int found = dictionary.indexOf(NullHandling.emptyToNullIfNeeded(endValue));
if (found >= 0) {
endIndex = endStrict ? found : found + 1;
} else {
endIndex = -(found + 1);
}
}
endIndex = Math.max(startIndex, endIndex);
return new IntIntImmutablePair(startIndex, endIndex);
}
};
}
}

464
processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java
View File

@ -22,13 +22,15 @@ package org.apache.druid.segment.virtual;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import org.apache.druid.collections.bitmap.BitmapFactory;
import com.google.common.base.Predicate;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
import org.apache.druid.query.dimension.DimensionSpec;
import org.apache.druid.query.dimension.ListFilteredDimensionSpec;
import org.apache.druid.query.filter.DruidPredicateFactory;
import org.apache.druid.query.ordering.StringComparators;
import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnSelector;
@ -37,10 +39,18 @@ import org.apache.druid.segment.ColumnValueSelector;
import org.apache.druid.segment.DimensionSelector;
import org.apache.druid.segment.IdMapping;
import org.apache.druid.segment.VirtualColumn;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexCapabilities;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex;
import org.apache.druid.segment.column.DruidPredicateIndex;
import org.apache.druid.segment.column.LexicographicalRangeIndex;
import org.apache.druid.segment.column.StringValueSetIndex;
import org.apache.druid.segment.filter.Filters;
import org.apache.druid.segment.serde.DictionaryEncodedStringIndexSupplier;
import javax.annotation.Nullable;
import java.util.Collections;
@ -50,7 +60,6 @@ import java.util.List;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Set;
import java.util.function.Predicate;
/**
* {@link VirtualColumn} form of {@link ListFilteredDimensionSpec}, powered by
@ -163,37 +172,53 @@ public class ListFilteredVirtualColumn implements VirtualColumn
return false;
}
@Nullable
@Override
public @Nullable BitmapIndex getBitmapIndex(
String columnName,
ColumnSelector selector
)
public ColumnIndexSupplier getIndexSupplier(String columnName, ColumnSelector columnSelector)
{
final ColumnHolder holder = selector.getColumnHolder(delegate.getDimension());
if (holder == null) {
return null;
}
final BitmapIndex underlyingIndex = holder.getBitmapIndex();
if (underlyingIndex == null) {
return null;
}
final IdMapping idMapping;
if (allowList) {
idMapping = ListFilteredDimensionSpec.buildAllowListIdMapping(
values,
underlyingIndex.getCardinality(),
null,
underlyingIndex::getValue
);
} else {
idMapping = ListFilteredDimensionSpec.buildDenyListIdMapping(
values,
underlyingIndex.getCardinality(),
underlyingIndex::getValue
);
}
return new ListFilteredBitmapIndex(underlyingIndex, idMapping);
return new ColumnIndexSupplier()
{
@Nullable
@Override
public <T> T as(Class<T> clazz)
{
final ColumnHolder holder = columnSelector.getColumnHolder(delegate.getDimension());
if (holder == null) {
return null;
}
DictionaryEncodedStringValueIndex underlyingIndex = holder.getIndexSupplier().as(
DictionaryEncodedStringValueIndex.class
);
if (underlyingIndex == null) {
return null;
}
final IdMapping idMapping;
if (allowList) {
idMapping = ListFilteredDimensionSpec.buildAllowListIdMapping(
values,
underlyingIndex.getCardinality(),
null,
underlyingIndex::getValue
);
} else {
idMapping = ListFilteredDimensionSpec.buildDenyListIdMapping(
values,
underlyingIndex.getCardinality(),
underlyingIndex::getValue
);
}
if (clazz.equals(StringValueSetIndex.class)) {
return (T) new ListFilteredStringValueSetIndex(underlyingIndex, idMapping);
} else if (clazz.equals(DruidPredicateIndex.class)) {
return (T) new ListFilteredDruidPredicateIndex(underlyingIndex, idMapping);
} else if (clazz.equals(LexicographicalRangeIndex.class)) {
return (T) new ListFilteredLexicographicalRangeIndex(underlyingIndex, idMapping);
} else if (clazz.equals(DictionaryEncodedStringValueIndex.class)) {
return (T) new ListFilteredDictionaryEncodedStringValueIndex(underlyingIndex, idMapping);
}
return null;
}
};
}
@Override
@ -227,107 +252,77 @@ public class ListFilteredVirtualColumn implements VirtualColumn
'}';
}
private static class ListFilteredBitmapIndex implements BitmapIndex
private abstract static class BaseVirtualIndex implements BitmapColumnIndex
{
final BitmapIndex delegate;
@Override
public ColumnIndexCapabilities getIndexCapabilities()
{
return DictionaryEncodedStringIndexSupplier.CAPABILITIES;
}
}
private static class BaseListFilteredColumnIndex
{
final DictionaryEncodedStringValueIndex delegate;
final IdMapping idMapping;
private ListFilteredBitmapIndex(BitmapIndex delegate, IdMapping idMapping)
private BaseListFilteredColumnIndex(
DictionaryEncodedStringValueIndex delegate,
IdMapping idMapping
)
{
this.delegate = delegate;
this.idMapping = idMapping;
}
@Override
public String getValue(int index)
{
return delegate.getValue(idMapping.getReverseId(index));
}
@Override
public boolean hasNulls()
{
return delegate.hasNulls();
}
@Override
public BitmapFactory getBitmapFactory()
{
return delegate.getBitmapFactory();
}
@Override
public int getCardinality()
{
return idMapping.getValueCardinality();
}
@Override
public int getIndex(@Nullable String value)
{
return getReverseIndex(value);
}
@Override
public ImmutableBitmap getBitmap(int idx)
ImmutableBitmap getBitmap(int idx)
{
return delegate.getBitmap(idMapping.getReverseId(idx));
}
@Override
public ImmutableBitmap getBitmapForValue(@Nullable String value)
int getCardinality()
{
if (getReverseIndex(value) < 0) {
return delegate.getBitmap(-1);
}
return delegate.getBitmapForValue(value);
return idMapping.getValueCardinality();
}
@Override
public Iterable<ImmutableBitmap> getBitmapsInRange(
@Nullable String startValue,
boolean startStrict,
@Nullable String endValue,
boolean endStrict,
Predicate<String> matcher
)
int getReverseIndex(@Nullable String value)
{
int startIndex, endIndex;
if (startValue == null) {
startIndex = 0;
} else {
final int found = getReverseIndex(NullHandling.emptyToNullIfNeeded(startValue));
if (found >= 0) {
startIndex = startStrict ? found + 1 : found;
int minIndex = 0;
int maxIndex = idMapping.getValueCardinality() - 1;
final Comparator<String> comparator = StringComparators.LEXICOGRAPHIC;
while (minIndex <= maxIndex) {
int currIndex = (minIndex + maxIndex) >>> 1;
String currValue = delegate.getValue(idMapping.getReverseId(currIndex));
int comparison = comparator.compare(currValue, value);
if (comparison == 0) {
return currIndex;
}
if (comparison < 0) {
minIndex = currIndex + 1;
} else {
startIndex = -(found + 1);
maxIndex = currIndex - 1;
}
}
if (endValue == null) {
endIndex = idMapping.getValueCardinality();
} else {
final int found = getReverseIndex(NullHandling.emptyToNullIfNeeded(endValue));
if (found >= 0) {
endIndex = endStrict ? found : found + 1;
} else {
endIndex = -(found + 1);
}
}
return -(minIndex + 1);
}
endIndex = startIndex > endIndex ? startIndex : endIndex;
final int start = startIndex, end = endIndex;
Iterable<ImmutableBitmap> getBitmapsInRange(Predicate<String> matcher, int start, int end)
{
return () -> new Iterator<ImmutableBitmap>()
{
int currIndex = start;
int found;
{
found = findNext();
}
private int findNext()
{
while (currIndex < end && !matcher.test(delegate.getValue(idMapping.getReverseId(currIndex)))) {
while (currIndex < end && !matcher.apply(delegate.getValue(idMapping.getReverseId(currIndex)))) {
currIndex++;
}
@ -358,70 +353,255 @@ public class ListFilteredVirtualColumn implements VirtualColumn
}
};
}
}
private static class ListFilteredStringValueSetIndex extends BaseListFilteredColumnIndex
implements StringValueSetIndex
{
private ListFilteredStringValueSetIndex(
DictionaryEncodedStringValueIndex delegate,
IdMapping idMapping
)
{
super(delegate, idMapping);
}
@Override
public Iterable<ImmutableBitmap> getBitmapsForValues(Set<String> values)
public BitmapColumnIndex forValue(@Nullable String value)
{
return () -> new Iterator<ImmutableBitmap>()
return new BaseVirtualIndex()
{
final Iterator<String> iterator = values.iterator();
int next = -1;
@Override
public boolean hasNext()
public double estimateSelectivity(int totalRows)
{
if (next < 0) {
findNext();
}
return next >= 0;
return Math.min(1, (double) getBitmapForValue().size() / totalRows);
}
@Override
public ImmutableBitmap next()
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory)
{
if (next < 0) {
findNext();
if (next < 0) {
throw new NoSuchElementException();
}
}
final int swap = next;
next = -1;
return getBitmap(swap);
return bitmapResultFactory.wrapDimensionValue(getBitmapForValue());
}
private void findNext()
private ImmutableBitmap getBitmapForValue()
{
while (next < 0 && iterator.hasNext()) {
String nextValue = iterator.next();
next = getReverseIndex(nextValue);
int reverseIndex = getReverseIndex(value);
if (reverseIndex < 0) {
return delegate.getBitmap(-1);
}
return delegate.getBitmap(idMapping.getReverseId(reverseIndex));
}
};
}
private int getReverseIndex(@Nullable String value)
@Override
public BitmapColumnIndex forValues(Set<String> values)
{
int minIndex = 0;
int maxIndex = idMapping.getValueCardinality() - 1;
final Comparator<String> comparator = StringComparators.LEXICOGRAPHIC;
while (minIndex <= maxIndex) {
int currIndex = (minIndex + maxIndex) >>> 1;
String currValue = delegate.getValue(idMapping.getReverseId(currIndex));
int comparison = comparator.compare(currValue, value);
if (comparison == 0) {
return currIndex;
return new BaseVirtualIndex()
{
@Override
public double estimateSelectivity(int totalRows)
{
return Filters.estimateSelectivity(getBitmapsIterable().iterator(), totalRows);
}
if (comparison < 0) {
minIndex = currIndex + 1;
} else {
maxIndex = currIndex - 1;
@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory)
{
return bitmapResultFactory.unionDimensionValueBitmaps(getBitmapsIterable());
}
}
return -(minIndex + 1);
private Iterable<ImmutableBitmap> getBitmapsIterable()
{
return () -> new Iterator<ImmutableBitmap>()
{
final Iterator<String> iterator = values.iterator();
int next = -1;
@Override
public boolean hasNext()
{
if (next < 0) {
findNext();
}
return next >= 0;
}
@Override
public ImmutableBitmap next()
{
if (next < 0) {
findNext();
if (next < 0) {
throw new NoSuchElementException();
}
}
final int swap = next;
next = -1;
return getBitmap(swap);
}
private void findNext()
{
while (next < 0 && iterator.hasNext()) {
String nextValue = iterator.next();
next = getReverseIndex(nextValue);
}
}
};
}
};
}
}
private static class ListFilteredDruidPredicateIndex extends BaseListFilteredColumnIndex
implements DruidPredicateIndex
{
private ListFilteredDruidPredicateIndex(DictionaryEncodedStringValueIndex delegate, IdMapping idMapping)
{
super(delegate, idMapping);
}
@Override
public BitmapColumnIndex forPredicate(DruidPredicateFactory matcherFactory)
{
return new BaseVirtualIndex()
{
@Override
public double estimateSelectivity(int totalRows)
{
final int start = 0, end = getCardinality();
return Filters.estimateSelectivity(
getBitmapsInRange(matcherFactory.makeStringPredicate(), start, end).iterator(),
totalRows
);
}
@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory)
{
final int start = 0, end = getCardinality();
return bitmapResultFactory.unionDimensionValueBitmaps(
getBitmapsInRange(matcherFactory.makeStringPredicate(), start, end)
);
}
};
}
}
private static class ListFilteredLexicographicalRangeIndex extends BaseListFilteredColumnIndex
implements LexicographicalRangeIndex
{
private ListFilteredLexicographicalRangeIndex(
DictionaryEncodedStringValueIndex delegate,
IdMapping idMapping
)
{
super(delegate, idMapping);
}
@Override
public BitmapColumnIndex forRange(
@Nullable String startValue,
boolean startStrict,
@Nullable String endValue,
boolean endStrict,
Predicate<String> matcher
)
{
return new BaseVirtualIndex()
{
@Override
public double estimateSelectivity(int totalRows)
{
return Filters.estimateSelectivity(getBitmapIterable().iterator(), totalRows);
}
@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory)
{
return bitmapResultFactory.unionDimensionValueBitmaps(getBitmapIterable());
}
private Iterable<ImmutableBitmap> getBitmapIterable()
{
int startIndex, endIndex;
if (startValue == null) {
startIndex = 0;
} else {
final int found = getReverseIndex(NullHandling.emptyToNullIfNeeded(startValue));
if (found >= 0) {
startIndex = startStrict ? found + 1 : found;
} else {
startIndex = -(found + 1);
}
}
if (endValue == null) {
endIndex = idMapping.getValueCardinality();
} else {
final int found = getReverseIndex(NullHandling.emptyToNullIfNeeded(endValue));
if (found >= 0) {
endIndex = endStrict ? found : found + 1;
} else {
endIndex = -(found + 1);
}
}
endIndex = Math.max(startIndex, endIndex);
final int start = startIndex, end = endIndex;
return getBitmapsInRange(matcher, start, end);
}
};
}
}
private static class ListFilteredDictionaryEncodedStringValueIndex extends BaseListFilteredColumnIndex
implements DictionaryEncodedStringValueIndex
{
private ListFilteredDictionaryEncodedStringValueIndex(
DictionaryEncodedStringValueIndex delegate,
IdMapping idMapping
)
{
super(delegate, idMapping);
}
@Override
public boolean hasNulls()
{
return delegate.hasNulls();
}
@Override
public int getCardinality()
{
return idMapping.getValueCardinality();
}
@Nullable
@Override
public String getValue(int index)
{
return delegate.getValue(idMapping.getReverseId(index));
}
@Override
public int getIndex(@Nullable String value)
{
return getReverseIndex(value);
}
@Override
public ImmutableBitmap getBitmap(int idx)
{
return delegate.getBitmap(idMapping.getReverseId(idx));
}
}
}

64
processing/src/test/java/org/apache/druid/segment/ColumnSelectorBitmapIndexSelectorTest.java → processing/src/test/java/org/apache/druid/segment/ColumnSelectorColumnIndexSelectorTest.java
View File

@ -21,18 +21,22 @@ package org.apache.druid.segment;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.query.DefaultBitmapResultFactory;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex;
import org.apache.druid.segment.column.StringDictionaryEncodedColumn;
import org.apache.druid.segment.data.Indexed;
import org.apache.druid.segment.column.StringValueSetIndex;
import org.apache.druid.segment.serde.NoIndexesColumnIndexSupplier;
import org.easymock.EasyMock;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
public class ColumnSelectorBitmapIndexSelectorTest
public class ColumnSelectorColumnIndexSelectorTest
{
private static final String STRING_DICTIONARY_COLUMN_NAME = "string";
private static final String NON_STRING_DICTIONARY_COLUMN_NAME = "not-string";
@ -41,7 +45,8 @@ public class ColumnSelectorBitmapIndexSelectorTest
VirtualColumns virtualColumns;
ColumnSelector index;
ColumnSelectorBitmapIndexSelector bitmapIndexSelector;
ColumnSelectorColumnIndexSelector indexSelector;
ColumnIndexSupplier indexSupplier;
@Before
public void setup()
@ -49,7 +54,8 @@ public class ColumnSelectorBitmapIndexSelectorTest
bitmapFactory = EasyMock.createMock(BitmapFactory.class);
virtualColumns = EasyMock.createMock(VirtualColumns.class);
index = EasyMock.createMock(ColumnSelector.class);
bitmapIndexSelector = new ColumnSelectorBitmapIndexSelector(bitmapFactory, virtualColumns, index);
indexSelector = new ColumnSelectorColumnIndexSelector(bitmapFactory, virtualColumns, index);
indexSupplier = EasyMock.createMock(ColumnIndexSupplier.class);
EasyMock.expect(virtualColumns.getVirtualColumn(STRING_DICTIONARY_COLUMN_NAME)).andReturn(null).anyTimes();
EasyMock.expect(virtualColumns.getVirtualColumn(NON_STRING_DICTIONARY_COLUMN_NAME)).andReturn(null).anyTimes();
@ -66,17 +72,22 @@ public class ColumnSelectorBitmapIndexSelectorTest
.setHasBitmapIndexes(true)
).anyTimes();
EasyMock.expect(holder.getColumn()).andReturn(stringColumn).anyTimes();
BitmapIndex someIndex = EasyMock.createMock(BitmapIndex.class);
EasyMock.expect(holder.getBitmapIndex()).andReturn(someIndex).anyTimes();
EasyMock.expect(holder.getIndexSupplier()).andReturn(indexSupplier).anyTimes();
StringValueSetIndex someIndex = EasyMock.createMock(StringValueSetIndex.class);
EasyMock.expect(indexSupplier.as(StringValueSetIndex.class)).andReturn(someIndex).anyTimes();
DictionaryEncodedStringValueIndex valueIndex = EasyMock.createMock(DictionaryEncodedStringValueIndex.class);
EasyMock.expect(indexSupplier.as(DictionaryEncodedStringValueIndex.class)).andReturn(valueIndex).anyTimes();
BitmapColumnIndex columnIndex = EasyMock.createMock(BitmapColumnIndex.class);
ImmutableBitmap someBitmap = EasyMock.createMock(ImmutableBitmap.class);
EasyMock.expect(someIndex.getIndex("foo")).andReturn(0).anyTimes();
EasyMock.expect(someIndex.getBitmap(0)).andReturn(someBitmap).anyTimes();
EasyMock.expect(someIndex.getBitmapForValue("foo")).andReturn(someBitmap).anyTimes();
EasyMock.expect(valueIndex.getIndex("foo")).andReturn(0).anyTimes();
EasyMock.expect(valueIndex.getBitmap(0)).andReturn(someBitmap).anyTimes();
EasyMock.expect(someIndex.forValue("foo")).andReturn(columnIndex).anyTimes();
EasyMock.expect(columnIndex.computeBitmapResult(EasyMock.anyObject())).andReturn(someBitmap).anyTimes();
ColumnHolder nonStringHolder = EasyMock.createMock(ColumnHolder.class);
EasyMock.expect(index.getColumnHolder(NON_STRING_DICTIONARY_COLUMN_NAME)).andReturn(nonStringHolder).anyTimes();
EasyMock.expect(nonStringHolder.getIndexSupplier()).andReturn(new NoIndexesColumnIndexSupplier()).anyTimes();
EasyMock.expect(nonStringHolder.getCapabilities()).andReturn(
ColumnCapabilitiesImpl.createDefault()
.setType(ColumnType.ofComplex("testBlob"))
@ -87,34 +98,39 @@ public class ColumnSelectorBitmapIndexSelectorTest
.setFilterable(true)
).anyTimes();
EasyMock.replay(bitmapFactory, virtualColumns, index, holder, stringColumn, nonStringHolder, someIndex, someBitmap);
EasyMock.replay(bitmapFactory, virtualColumns, index, indexSupplier, holder, stringColumn, nonStringHolder, someIndex, columnIndex, valueIndex, someBitmap);
}
@Test
public void testStringDictionaryUseIndex()
{
BitmapIndex bitmapIndex = bitmapIndexSelector.getBitmapIndex(STRING_DICTIONARY_COLUMN_NAME);
final ColumnIndexSupplier supplier = indexSelector.getIndexSupplier(STRING_DICTIONARY_COLUMN_NAME);
DictionaryEncodedStringValueIndex bitmapIndex = supplier.as(
DictionaryEncodedStringValueIndex.class
);
Assert.assertNotNull(bitmapIndex);
Indexed<String> vals = bitmapIndexSelector.getDimensionValues(STRING_DICTIONARY_COLUMN_NAME);
Assert.assertNotNull(vals);
ImmutableBitmap valueIndex = bitmapIndexSelector.getBitmapIndex(STRING_DICTIONARY_COLUMN_NAME, "foo");
StringValueSetIndex valueIndex = supplier.as(StringValueSetIndex.class);
Assert.assertNotNull(valueIndex);
EasyMock.verify(bitmapFactory, virtualColumns, index);
ImmutableBitmap valueBitmap = valueIndex.forValue("foo")
.computeBitmapResult(
new DefaultBitmapResultFactory(indexSelector.getBitmapFactory())
);
Assert.assertNotNull(valueBitmap);
EasyMock.verify(bitmapFactory, virtualColumns, index, indexSupplier);
}
@Test
public void testNonStringDictionaryDoNotUseIndex()
{
BitmapIndex bitmapIndex = bitmapIndexSelector.getBitmapIndex(NON_STRING_DICTIONARY_COLUMN_NAME);
final ColumnIndexSupplier supplier = indexSelector.getIndexSupplier(NON_STRING_DICTIONARY_COLUMN_NAME);
DictionaryEncodedStringValueIndex bitmapIndex = supplier.as(
DictionaryEncodedStringValueIndex.class
);
Assert.assertNull(bitmapIndex);
Indexed<String> vals = bitmapIndexSelector.getDimensionValues(NON_STRING_DICTIONARY_COLUMN_NAME);
Assert.assertNull(vals);
ImmutableBitmap valueIndex = bitmapIndexSelector.getBitmapIndex(NON_STRING_DICTIONARY_COLUMN_NAME, "foo");
StringValueSetIndex valueIndex = supplier.as(StringValueSetIndex.class);
Assert.assertNull(valueIndex);
EasyMock.verify(bitmapFactory, virtualColumns, index);
EasyMock.verify(bitmapFactory, virtualColumns, index, indexSupplier);
}
}

25
processing/src/test/java/org/apache/druid/segment/IndexMergerNullHandlingTest.java
View File

@ -28,10 +28,12 @@ import org.apache.druid.common.config.NullHandling;
import org.apache.druid.data.input.MapBasedInputRow;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.guava.Comparators;
import org.apache.druid.query.DefaultBitmapResultFactory;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.DictionaryEncodedColumn;
import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex;
import org.apache.druid.segment.column.StringValueSetIndex;
import org.apache.druid.segment.data.IncrementalIndexTest;
import org.apache.druid.segment.data.IndexedInts;
import org.apache.druid.segment.incremental.IncrementalIndex;
@ -180,7 +182,12 @@ public class IndexMergerNullHandlingTest
);
// Verify that the bitmap index for null is correct.
final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
final DictionaryEncodedStringValueIndex valueIndex = columnHolder.getIndexSupplier().as(
DictionaryEncodedStringValueIndex.class
);
final StringValueSetIndex valueSetIndex = columnHolder.getIndexSupplier().as(
StringValueSetIndex.class
);
// Read through the column to find all the rows that should match null.
final List<Integer> expectedNullRows = new ArrayList<>();
@ -191,12 +198,18 @@ public class IndexMergerNullHandlingTest
}
}
Assert.assertEquals(subsetList.toString(), expectedNullRows.size() > 0, bitmapIndex.hasNulls());
Assert.assertEquals(subsetList.toString(), expectedNullRows.size() > 0, valueIndex.hasNulls());
if (expectedNullRows.size() > 0) {
Assert.assertEquals(subsetList.toString(), 0, bitmapIndex.getIndex(null));
Assert.assertEquals(subsetList.toString(), 0, valueIndex.getIndex(null));
final ImmutableBitmap nullBitmap = bitmapIndex.getBitmapForValue(null);
final ImmutableBitmap nullBitmap = valueSetIndex.forValue(null)
.computeBitmapResult(
new DefaultBitmapResultFactory(
indexSpec.getBitmapSerdeFactory()
.getBitmapFactory()
)
);
final List<Integer> actualNullRows = new ArrayList<>();
final IntIterator iterator = nullBitmap.iterator();
while (iterator.hasNext()) {
@ -205,7 +218,7 @@ public class IndexMergerNullHandlingTest
Assert.assertEquals(subsetList.toString(), expectedNullRows, actualNullRows);
} else {
Assert.assertEquals(-1, bitmapIndex.getIndex(null));
Assert.assertEquals(-1, valueIndex.getIndex(null));
}
}
}

331
processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java
View File

@ -41,17 +41,21 @@ import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.io.smoosh.SmooshedFileMapper;
import org.apache.druid.query.DefaultBitmapResultFactory;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.DictionaryEncodedColumn;
import org.apache.druid.segment.column.StringDictionaryEncodedColumn;
import org.apache.druid.segment.column.StringValueSetIndex;
import org.apache.druid.segment.data.BitmapSerdeFactory;
import org.apache.druid.segment.data.BitmapValues;
import org.apache.druid.segment.data.CompressionFactory;
import org.apache.druid.segment.data.CompressionStrategy;
import org.apache.druid.segment.data.ConciseBitmapSerdeFactory;
import org.apache.druid.segment.data.ImmutableBitmapValues;
import org.apache.druid.segment.data.IncrementalIndexTest;
import org.apache.druid.segment.incremental.IncrementalIndex;
import org.apache.druid.segment.incremental.IncrementalIndexAdapter;
@ -129,6 +133,29 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
}
}
static BitmapValues getBitmapIndex(QueryableIndexIndexableAdapter adapter, String dimension, String value)
{
final ColumnHolder columnHolder = adapter.getQueryableIndex().getColumnHolder(dimension);
if (columnHolder == null) {
return BitmapValues.EMPTY;
}
final ColumnIndexSupplier indexSupplier = columnHolder.getIndexSupplier();
if (indexSupplier == null) {
return BitmapValues.EMPTY;
}
final StringValueSetIndex index = indexSupplier.as(StringValueSetIndex.class);
if (index == null) {
return BitmapValues.EMPTY;
}
return new ImmutableBitmapValues(index.forValue(value).computeBitmapResult(
new DefaultBitmapResultFactory(adapter.getQueryableIndex().getBitmapFactoryForDimensions()))
);
}
private final IndexSpec indexSpec;
private final IndexIO indexIO;
private final boolean useBitmapIndexes;
@ -219,12 +246,12 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
Assert.assertEquals(ImmutableList.of("1", "2"), rowList.get(0).dimensionValues());
Assert.assertEquals(Arrays.asList("3", null), rowList.get(1).dimensionValues());
checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("dim1", null));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dim1", "1"));
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("dim1", "3"));
checkBitmapIndex(Collections.emptyList(), getBitmapIndex(adapter, "dim1", null));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "dim1", "1"));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "dim1", "3"));
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("dim2", null));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dim2", "2"));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "dim2", null));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "dim2", "2"));
}
@Test
@ -618,20 +645,20 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
Assert.assertEquals(Arrays.asList("50000", "200", "3000"), rowList.get(2).dimensionValues());
Assert.assertEquals(Collections.singletonList(3L), rowList.get(2).metricValues());
checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("d3", null));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d3", "30000"));
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d3", "40000"));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d3", "50000"));
checkBitmapIndex(Collections.emptyList(), getBitmapIndex(adapter, "d3", null));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "d3", "30000"));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "d3", "40000"));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "d3", "50000"));
checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("d1", null));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d1", "100"));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d1", "200"));
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d1", "300"));
checkBitmapIndex(Collections.emptyList(), getBitmapIndex(adapter, "d1", null));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "d1", "100"));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "d1", "200"));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "d1", "300"));
checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("d2", null));
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d2", "2000"));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d2", "3000"));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d2", "4000"));
checkBitmapIndex(Collections.emptyList(), getBitmapIndex(adapter, "d2", null));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "d2", "2000"));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "d2", "3000"));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "d2", "4000"));
}
@ -715,18 +742,18 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dimC").hasBitmapIndexes());
if (useBitmapIndexes) {
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dimA", null));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("dimA", "1"));
checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("dimA", "2"));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "dimA", null));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "dimA", "1"));
checkBitmapIndex(Collections.singletonList(3), getBitmapIndex(adapter, "dimA", "2"));
checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("dimB", null));
checkBitmapIndex(Collections.emptyList(), getBitmapIndex(adapter, "dimB", null));
checkBitmapIndex(Arrays.asList(2, 3), adapter.getBitmapIndex("dimC", null));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dimC", "1"));
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("dimC", "2"));
checkBitmapIndex(Arrays.asList(2, 3), getBitmapIndex(adapter, "dimC", null));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "dimC", "1"));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "dimC", "2"));
}
checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("dimB", ""));
checkBitmapIndex(Collections.emptyList(), getBitmapIndex(adapter, "dimB", ""));
}
@ -790,9 +817,9 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
// dimA always has bitmap indexes, since it has them in indexA (it comes in through discovery).
Assert.assertTrue(adapter.getCapabilities("dimA").hasBitmapIndexes());
checkBitmapIndex(Arrays.asList(0, 1, 2), adapter.getBitmapIndex("dimA", null));
checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("dimA", "1"));
checkBitmapIndex(Collections.singletonList(4), adapter.getBitmapIndex("dimA", "2"));
checkBitmapIndex(Arrays.asList(0, 1, 2), getBitmapIndex(adapter, "dimA", null));
checkBitmapIndex(Collections.singletonList(3), getBitmapIndex(adapter, "dimA", "1"));
checkBitmapIndex(Collections.singletonList(4), getBitmapIndex(adapter, "dimA", "2"));
// dimB may or may not have bitmap indexes, since it comes in through explicit definition in toPersistB2.
@ -802,10 +829,10 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
}
//noinspection ObjectEquality
if (toPersistB != toPersistB2 || useBitmapIndexes) {
checkBitmapIndex(Arrays.asList(3, 4), adapter.getBitmapIndex("dimB", null));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dimB", "1"));
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("dimB", "2"));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("dimB", "3"));
checkBitmapIndex(Arrays.asList(3, 4), getBitmapIndex(adapter, "dimB", null));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "dimB", "1"));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "dimB", "2"));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "dimB", "3"));
}
}
}
@ -930,9 +957,9 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
rowList.get(3).dimensionValues()
);
checkBitmapIndex(Arrays.asList(0, 2, 3), adapter.getBitmapIndex("d2", null));
checkBitmapIndex(Arrays.asList(0, 1, 3), adapter.getBitmapIndex("d5", null));
checkBitmapIndex(Arrays.asList(0, 3), adapter.getBitmapIndex("d7", null));
checkBitmapIndex(Arrays.asList(0, 2, 3), getBitmapIndex(adapter, "d2", null));
checkBitmapIndex(Arrays.asList(0, 1, 3), getBitmapIndex(adapter, "d5", null));
checkBitmapIndex(Arrays.asList(0, 3), getBitmapIndex(adapter, "d7", null));
} else {
Assert.assertEquals(
Arrays.asList("", "", "310", null, null, "", null, "910"),
@ -950,36 +977,36 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
Arrays.asList(null, null, null, "", "621", "", "821", "921"),
rowList.get(3).dimensionValues()
);
checkBitmapIndex(Arrays.asList(2, 3), adapter.getBitmapIndex("d2", null));
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("d5", null));
checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("d7", null));
checkBitmapIndex(Arrays.asList(2, 3), getBitmapIndex(adapter, "d2", null));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "d5", null));
checkBitmapIndex(Collections.emptyList(), getBitmapIndex(adapter, "d7", null));
}
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d2", "210"));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "d2", "210"));
checkBitmapIndex(Arrays.asList(2, 3), adapter.getBitmapIndex("d3", null));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d3", "310"));
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d3", "311"));
checkBitmapIndex(Arrays.asList(2, 3), getBitmapIndex(adapter, "d3", null));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "d3", "310"));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "d3", "311"));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d5", "520"));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "d5", "520"));
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("d6", null));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d6", "620"));
checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("d6", "621"));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "d6", null));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "d6", "620"));
checkBitmapIndex(Collections.singletonList(3), getBitmapIndex(adapter, "d6", "621"));
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d7", "710"));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d7", "720"));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "d7", "710"));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "d7", "720"));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d8", null));
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d8", "810"));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d8", "820"));
checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("d8", "821"));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "d8", null));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "d8", "810"));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "d8", "820"));
checkBitmapIndex(Collections.singletonList(3), getBitmapIndex(adapter, "d8", "821"));
checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("d9", null));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d9", "910"));
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d9", "911"));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d9", "920"));
checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("d9", "921"));
checkBitmapIndex(Collections.emptyList(), getBitmapIndex(adapter, "d9", null));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "d9", "910"));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "d9", "911"));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "d9", "920"));
checkBitmapIndex(Collections.singletonList(3), getBitmapIndex(adapter, "d9", "921"));
}
}
@ -1102,18 +1129,18 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
);
}
checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("d3", null));
checkBitmapIndex(Arrays.asList(0, 1, 2), adapter.getBitmapIndex("d3", "310"));
checkBitmapIndex(Collections.singletonList(3), getBitmapIndex(adapter, "d3", null));
checkBitmapIndex(Arrays.asList(0, 1, 2), getBitmapIndex(adapter, "d3", "310"));
checkBitmapIndex(Arrays.asList(0, 1, 2), adapter.getBitmapIndex("d6", null));
checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("d6", "621"));
checkBitmapIndex(Arrays.asList(0, 1, 2), getBitmapIndex(adapter, "d6", null));
checkBitmapIndex(Collections.singletonList(3), getBitmapIndex(adapter, "d6", "621"));
checkBitmapIndex(Arrays.asList(0, 1, 2), adapter.getBitmapIndex("d8", null));
checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("d8", "821"));
checkBitmapIndex(Arrays.asList(0, 1, 2), getBitmapIndex(adapter, "d8", null));
checkBitmapIndex(Collections.singletonList(3), getBitmapIndex(adapter, "d8", "821"));
checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("d9", null));
checkBitmapIndex(Arrays.asList(0, 1, 2), adapter.getBitmapIndex("d9", "910"));
checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("d9", "921"));
checkBitmapIndex(Collections.emptyList(), getBitmapIndex(adapter, "d9", null));
checkBitmapIndex(Arrays.asList(0, 1, 2), getBitmapIndex(adapter, "d9", "910"));
checkBitmapIndex(Collections.singletonList(3), getBitmapIndex(adapter, "d9", "921"));
}
private void checkBitmapIndex(List<Integer> expected, BitmapValues real)
@ -1240,14 +1267,14 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
Assert.assertEquals(Arrays.asList("3", null), rowList.get(4).dimensionValues());
Assert.assertEquals(Collections.singletonList(2L), rowList.get(4).metricValues());
checkBitmapIndex(Arrays.asList(2, 3, 4), adapter.getBitmapIndex("dimA", null));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dimA", "1"));
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("dimA", "2"));
checkBitmapIndex(Arrays.asList(2, 3, 4), getBitmapIndex(adapter, "dimA", null));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "dimA", "1"));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "dimA", "2"));
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dimB", null));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("dimB", "1"));
checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("dimB", "2"));
checkBitmapIndex(Collections.singletonList(4), adapter.getBitmapIndex("dimB", "3"));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "dimB", null));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "dimB", "1"));
checkBitmapIndex(Collections.singletonList(3), getBitmapIndex(adapter, "dimB", "2"));
checkBitmapIndex(Collections.singletonList(4), getBitmapIndex(adapter, "dimB", "3"));
Assert.assertEquals(ImmutableList.of("dimA", "dimB", "dimC"), ImmutableList.copyOf(adapter2.getDimensionNames()));
@ -1282,19 +1309,19 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
Assert.assertEquals(Arrays.asList("2", null, null), rowList2.get(11).dimensionValues());
Assert.assertEquals(Collections.singletonList(2L), rowList2.get(11).metricValues());
checkBitmapIndex(Arrays.asList(0, 1, 2, 3, 4, 5, 8, 9, 10), adapter2.getBitmapIndex("dimA", null));
checkBitmapIndex(Collections.singletonList(6), adapter2.getBitmapIndex("dimA", "1"));
checkBitmapIndex(Arrays.asList(7, 11), adapter2.getBitmapIndex("dimA", "2"));
checkBitmapIndex(Arrays.asList(0, 1, 2, 3, 4, 5, 8, 9, 10), getBitmapIndex(adapter2, "dimA", null));
checkBitmapIndex(Collections.singletonList(6), getBitmapIndex(adapter2, "dimA", "1"));
checkBitmapIndex(Arrays.asList(7, 11), getBitmapIndex(adapter2, "dimA", "2"));
checkBitmapIndex(Arrays.asList(0, 1, 2, 6, 7, 11), adapter2.getBitmapIndex("dimB", null));
checkBitmapIndex(Arrays.asList(3, 8), adapter2.getBitmapIndex("dimB", "1"));
checkBitmapIndex(Arrays.asList(4, 9), adapter2.getBitmapIndex("dimB", "2"));
checkBitmapIndex(Arrays.asList(5, 10), adapter2.getBitmapIndex("dimB", "3"));
checkBitmapIndex(Arrays.asList(0, 1, 2, 6, 7, 11), getBitmapIndex(adapter2, "dimB", null));
checkBitmapIndex(Arrays.asList(3, 8), getBitmapIndex(adapter2, "dimB", "1"));
checkBitmapIndex(Arrays.asList(4, 9), getBitmapIndex(adapter2, "dimB", "2"));
checkBitmapIndex(Arrays.asList(5, 10), getBitmapIndex(adapter2, "dimB", "3"));
checkBitmapIndex(Arrays.asList(3, 4, 5, 6, 7, 8, 9, 10, 11), adapter2.getBitmapIndex("dimC", null));
checkBitmapIndex(Collections.singletonList(0), adapter2.getBitmapIndex("dimC", "1"));
checkBitmapIndex(Collections.singletonList(1), adapter2.getBitmapIndex("dimC", "2"));
checkBitmapIndex(Collections.singletonList(2), adapter2.getBitmapIndex("dimC", "3"));
checkBitmapIndex(Arrays.asList(3, 4, 5, 6, 7, 8, 9, 10, 11), getBitmapIndex(adapter2, "dimC", null));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter2, "dimC", "1"));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter2, "dimC", "2"));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter2, "dimC", "3"));
}
@ -1865,14 +1892,14 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim2").hasBitmapIndexes());
if (useBitmapIndexes) {
checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("dim1", null));
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dim1", "a"));
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dim1", "b"));
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dim1", "x"));
checkBitmapIndex(Collections.emptyList(), getBitmapIndex(adapter, "dim1", null));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "dim1", "a"));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "dim1", "b"));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "dim1", "x"));
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dim2", "a"));
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dim2", "b"));
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dim2", "x"));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "dim2", "a"));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "dim2", "b"));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "dim2", "x"));
}
// xaab-axbx + abx-xab --> abx-abx + abx-abx --> abx-abx
@ -1896,14 +1923,14 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim2").hasBitmapIndexes());
if (useBitmapIndexes) {
checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("dim1", null));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dim1", "a"));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dim1", "b"));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dim1", "x"));
checkBitmapIndex(Collections.emptyList(), getBitmapIndex(adapter, "dim1", null));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "dim1", "a"));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "dim1", "b"));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "dim1", "x"));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dim2", "a"));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dim2", "b"));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dim2", "x"));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "dim2", "a"));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "dim2", "b"));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "dim2", "x"));
}
// xaab-axbx + abx-xab --> abx-xab + xaab-axbx
@ -1931,14 +1958,14 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim2").hasBitmapIndexes());
if (useBitmapIndexes) {
checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("dim1", null));
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dim1", "a"));
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dim1", "b"));
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dim1", "x"));
checkBitmapIndex(Collections.emptyList(), getBitmapIndex(adapter, "dim1", null));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "dim1", "a"));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "dim1", "b"));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "dim1", "x"));
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dim2", "a"));
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dim2", "b"));
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dim2", "x"));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "dim2", "a"));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "dim2", "b"));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "dim2", "x"));
}
}
@ -2090,15 +2117,15 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
Assert.assertEquals(Arrays.asList("potato", Arrays.asList("0", "1", "4")), rowList.get(2).dimensionValues());
Assert.assertEquals(1L, rowList.get(2).metricValues().get(0));
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dimA", "leek"));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("dimA", "potato"));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "dimA", "leek"));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "dimA", "potato"));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("dimMultiVal", "0"));
checkBitmapIndex(Arrays.asList(0, 1, 2), adapter.getBitmapIndex("dimMultiVal", "1"));
checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dimMultiVal", "2"));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dimMultiVal", "3"));
checkBitmapIndex(Arrays.asList(1, 2), adapter.getBitmapIndex("dimMultiVal", "4"));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dimMultiVal", "5"));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "dimMultiVal", "0"));
checkBitmapIndex(Arrays.asList(0, 1, 2), getBitmapIndex(adapter, "dimMultiVal", "1"));
checkBitmapIndex(Arrays.asList(0, 1), getBitmapIndex(adapter, "dimMultiVal", "2"));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "dimMultiVal", "3"));
checkBitmapIndex(Arrays.asList(1, 2), getBitmapIndex(adapter, "dimMultiVal", "4"));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "dimMultiVal", "5"));
}
@ -2348,16 +2375,16 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
Assert.assertEquals(Arrays.asList("potato", "2"), rowList.get(10).dimensionValues());
Assert.assertEquals(4L, rowList.get(10).metricValues().get(0));
checkBitmapIndex(Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8), adapter.getBitmapIndex("dimA", "leek"));
checkBitmapIndex(Arrays.asList(9, 10), adapter.getBitmapIndex("dimA", "potato"));
checkBitmapIndex(Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8), getBitmapIndex(adapter, "dimA", "leek"));
checkBitmapIndex(Arrays.asList(9, 10), getBitmapIndex(adapter, "dimA", "potato"));
checkBitmapIndex(Arrays.asList(0, 1, 2), adapter.getBitmapIndex("dimMultiVal", null));
checkBitmapIndex(ImmutableList.of(), adapter.getBitmapIndex("dimMultiVal", ""));
checkBitmapIndex(Arrays.asList(1, 3, 4, 5, 6, 7, 9), adapter.getBitmapIndex("dimMultiVal", "1"));
checkBitmapIndex(Arrays.asList(1, 4, 8, 10), adapter.getBitmapIndex("dimMultiVal", "2"));
checkBitmapIndex(Arrays.asList(1, 2, 4, 5, 6, 9), adapter.getBitmapIndex("dimMultiVal", "3"));
checkBitmapIndex(Collections.singletonList(7), adapter.getBitmapIndex("dimMultiVal", "4"));
checkBitmapIndex(Collections.singletonList(6), adapter.getBitmapIndex("dimMultiVal", "5"));
checkBitmapIndex(Arrays.asList(0, 1, 2), getBitmapIndex(adapter, "dimMultiVal", null));
checkBitmapIndex(ImmutableList.of(), getBitmapIndex(adapter, "dimMultiVal", ""));
checkBitmapIndex(Arrays.asList(1, 3, 4, 5, 6, 7, 9), getBitmapIndex(adapter, "dimMultiVal", "1"));
checkBitmapIndex(Arrays.asList(1, 4, 8, 10), getBitmapIndex(adapter, "dimMultiVal", "2"));
checkBitmapIndex(Arrays.asList(1, 2, 4, 5, 6, 9), getBitmapIndex(adapter, "dimMultiVal", "3"));
checkBitmapIndex(Collections.singletonList(7), getBitmapIndex(adapter, "dimMultiVal", "4"));
checkBitmapIndex(Collections.singletonList(6), getBitmapIndex(adapter, "dimMultiVal", "5"));
} else {
Assert.assertEquals(14, rowList.size());
@ -2403,16 +2430,16 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
Assert.assertEquals(Arrays.asList("potato", "2"), rowList.get(13).dimensionValues());
Assert.assertEquals(4L, rowList.get(13).metricValues().get(0));
checkBitmapIndex(Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11), adapter.getBitmapIndex("dimA", "leek"));
checkBitmapIndex(Arrays.asList(12, 13), adapter.getBitmapIndex("dimA", "potato"));
checkBitmapIndex(Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11), getBitmapIndex(adapter, "dimA", "leek"));
checkBitmapIndex(Arrays.asList(12, 13), getBitmapIndex(adapter, "dimA", "potato"));
checkBitmapIndex(Arrays.asList(0, 1, 2), adapter.getBitmapIndex("dimMultiVal", null));
checkBitmapIndex(ImmutableList.of(3, 4, 5), adapter.getBitmapIndex("dimMultiVal", ""));
checkBitmapIndex(Arrays.asList(1, 4, 6, 7, 8, 9, 10, 12), adapter.getBitmapIndex("dimMultiVal", "1"));
checkBitmapIndex(Arrays.asList(1, 4, 7, 11, 13), adapter.getBitmapIndex("dimMultiVal", "2"));
checkBitmapIndex(Arrays.asList(1, 2, 4, 5, 7, 8, 9, 12), adapter.getBitmapIndex("dimMultiVal", "3"));
checkBitmapIndex(Collections.singletonList(10), adapter.getBitmapIndex("dimMultiVal", "4"));
checkBitmapIndex(Collections.singletonList(9), adapter.getBitmapIndex("dimMultiVal", "5"));
checkBitmapIndex(Arrays.asList(0, 1, 2), getBitmapIndex(adapter, "dimMultiVal", null));
checkBitmapIndex(ImmutableList.of(3, 4, 5), getBitmapIndex(adapter, "dimMultiVal", ""));
checkBitmapIndex(Arrays.asList(1, 4, 6, 7, 8, 9, 10, 12), getBitmapIndex(adapter, "dimMultiVal", "1"));
checkBitmapIndex(Arrays.asList(1, 4, 7, 11, 13), getBitmapIndex(adapter, "dimMultiVal", "2"));
checkBitmapIndex(Arrays.asList(1, 2, 4, 5, 7, 8, 9, 12), getBitmapIndex(adapter, "dimMultiVal", "3"));
checkBitmapIndex(Collections.singletonList(10), getBitmapIndex(adapter, "dimMultiVal", "4"));
checkBitmapIndex(Collections.singletonList(9), getBitmapIndex(adapter, "dimMultiVal", "5"));
}
}
@ -2474,31 +2501,31 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest
);
Assert.assertEquals(3L, rowList.get(3).metricValues().get(0));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d1", "a"));
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d1", "aa"));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d1", "aaa"));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d1", "aaa"));
checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("d1", "1"));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "d1", "a"));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "d1", "aa"));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "d1", "aaa"));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "d1", "aaa"));
checkBitmapIndex(Collections.singletonList(3), getBitmapIndex(adapter, "d1", "1"));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d2", "b"));
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d2", "bb"));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d2", "bbb"));
checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("d2", "2"));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "d2", "b"));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "d2", "bb"));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "d2", "bbb"));
checkBitmapIndex(Collections.singletonList(3), getBitmapIndex(adapter, "d2", "2"));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d3", "c"));
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d3", "cc"));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d3", "ccc"));
checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("d3", "3"));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "d3", "c"));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "d3", "cc"));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "d3", "ccc"));
checkBitmapIndex(Collections.singletonList(3), getBitmapIndex(adapter, "d3", "3"));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d4", "d"));
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d4", "dd"));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d4", "ddd"));
checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("d4", "4"));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "d4", "d"));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "d4", "dd"));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "d4", "ddd"));
checkBitmapIndex(Collections.singletonList(3), getBitmapIndex(adapter, "d4", "4"));
checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d5", "e"));
checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d5", "ee"));
checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d5", "eee"));
checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("d5", "5"));
checkBitmapIndex(Collections.singletonList(0), getBitmapIndex(adapter, "d5", "e"));
checkBitmapIndex(Collections.singletonList(1), getBitmapIndex(adapter, "d5", "ee"));
checkBitmapIndex(Collections.singletonList(2), getBitmapIndex(adapter, "d5", "eee"));
checkBitmapIndex(Collections.singletonList(3), getBitmapIndex(adapter, "d5", "5"));
}
@Test

60
processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
View File

@ -50,14 +50,13 @@ import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprType;
import org.apache.druid.math.expr.ExpressionType;
import org.apache.druid.math.expr.Parser;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.aggregation.Aggregator;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.FilteredAggregatorFactory;
import org.apache.druid.query.aggregation.VectorAggregator;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
import org.apache.druid.query.expression.TestExprMacroTable;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.DimFilter;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.ValueMatcher;
@ -76,6 +75,7 @@ import org.apache.druid.segment.RowBasedColumnSelectorFactory;
import org.apache.druid.segment.RowBasedStorageAdapter;
import org.apache.druid.segment.StorageAdapter;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.data.BitmapSerdeFactory;
@ -517,11 +517,6 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
final Filter theFilter = makeFilter(filter);
final Filter postFilteringFilter = new Filter()
{
@Override
public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory)
{
throw new UnsupportedOperationException();
}
@Override
public ValueMatcher makeMatcher(ColumnSelectorFactory factory)
@ -530,19 +525,7 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return false;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return false;
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, ColumnIndexSelector indexSelector)
{
return false;
}
@ -554,10 +537,17 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
public double estimateSelectivity(ColumnIndexSelector indexSelector)
{
return 1.0;
}
@Nullable
@Override
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
return null;
}
};
final Sequence<Cursor> cursors = makeCursorSequence(postFilteringFilter);
@ -591,11 +581,6 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
final Filter theFilter = makeFilter(filter);
final Filter postFilteringFilter = new Filter()
{
@Override
public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory)
{
throw new UnsupportedOperationException();
}
@Override
public ValueMatcher makeMatcher(ColumnSelectorFactory factory)
@ -603,18 +588,6 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
return theFilter.makeMatcher(factory);
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return false;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return false;
}
@Override
public VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory factory)
{
@ -634,16 +607,23 @@ public abstract class BaseFilterTest extends InitializedNullHandlingTest
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, ColumnIndexSelector indexSelector)
{
return false;
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
public double estimateSelectivity(ColumnIndexSelector indexSelector)
{
return 1.0;
}
@Nullable
@Override
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
return null;
}
};
try (final VectorCursor cursor = makeVectorCursor(postFilteringFilter)) {

172
processing/src/test/java/org/apache/druid/segment/filter/ExtractionDimFilterTest.java
View File

@ -26,27 +26,23 @@ import org.apache.druid.collections.bitmap.ConciseBitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.collections.bitmap.MutableBitmap;
import org.apache.druid.collections.bitmap.RoaringBitmapFactory;
import org.apache.druid.collections.spatial.ImmutableRTree;
import org.apache.druid.query.extraction.DimExtractionFn;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.DimFilters;
import org.apache.druid.query.filter.ExtractionDimFilter;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.SelectorDimFilter;
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.data.BitmapSerdeFactory;
import org.apache.druid.segment.data.CloseableIndexed;
import org.apache.druid.segment.data.ConciseBitmapSerdeFactory;
import org.apache.druid.segment.data.GenericIndexed;
import org.apache.druid.segment.data.Indexed;
import org.apache.druid.segment.data.ListIndexed;
import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
import org.apache.druid.segment.serde.StringBitmapIndexColumnPartSupplier;
import org.apache.druid.segment.serde.DictionaryEncodedStringIndexSupplier;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -54,21 +50,14 @@ import org.junit.runners.Parameterized;
import javax.annotation.Nullable;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;
/**
*
*/
@RunWith(Parameterized.class)
public class ExtractionDimFilterTest
public class ExtractionDimFilterTest extends InitializedNullHandlingTest
{
private static final Map<String, String[]> DIM_VALS = ImmutableMap.of(
"foo", new String[]{"foo1", "foo2", "foo3"},
"bar", new String[]{"bar1"},
"baz", new String[]{"foo1"}
);
private static final Map<String, String> EXTRACTION_VALUES = ImmutableMap.of(
"foo1", "extractDimVal"
);
@ -95,70 +84,18 @@ public class ExtractionDimFilterTest
private final BitmapSerdeFactory serdeFactory;
private final ImmutableBitmap foo1BitMap;
private final BitmapIndexSelector BITMAP_INDEX_SELECTOR = new BitmapIndexSelector()
private final ColumnIndexSelector BITMAP_INDEX_SELECTOR = new ColumnIndexSelector()
{
@Nullable
@Override
public ColumnCapabilities getColumnCapabilities(String column)
{
return ColumnCapabilitiesImpl.createDefault().setType(ColumnType.STRING).setHasMultipleValues(true);
}
@Override
public CloseableIndexed<String> getDimensionValues(String dimension)
{
final String[] vals = DIM_VALS.get(dimension);
if (vals == null) {
return null;
} else {
Indexed<String> indexed = new ListIndexed<>(vals);
return new CloseableIndexed<String>()
{
@Override
public int size()
{
return indexed.size();
}
@Nullable
@Override
public String get(int index)
{
return indexed.get(index);
}
@Override
public int indexOf(@Nullable String value)
{
return indexed.indexOf(value);
}
@Override
public void inspectRuntimeShape(RuntimeShapeInspector inspector)
{
inspector.visit("indexed", indexed);
}
@Override
public void close()
{
// close nothing
}
@Override
public Iterator<String> iterator()
{
return indexed.iterator();
}
};
}
}
@Override
public ColumnCapabilities.Capable hasMultipleValues(final String dimension)
{
return ColumnCapabilities.Capable.TRUE;
return ColumnCapabilitiesImpl.createDefault()
.setType(ColumnType.STRING)
.setHasMultipleValues(true)
.setDictionaryEncoded(true)
.setDictionaryValuesUnique(true)
.setDictionaryValuesSorted(true);
}
@Override
@ -174,26 +111,19 @@ public class ExtractionDimFilterTest
}
@Override
public ImmutableBitmap getBitmapIndex(String dimension, String value)
{
return "foo1".equals(value) ? foo1BitMap : null;
}
@Override
public BitmapIndex getBitmapIndex(String dimension)
{
return new StringBitmapIndexColumnPartSupplier(
factory,
GenericIndexed.fromIterable(Collections.singletonList(foo1BitMap), serdeFactory.getObjectStrategy()),
GenericIndexed.fromIterable(Collections.singletonList("foo1"), GenericIndexed.STRING_STRATEGY)
).get();
}
@Override
public ImmutableRTree getSpatialIndex(String dimension)
public ColumnIndexSupplier getIndexSupplier(String column)
{
if ("foo".equals(column)) {
return new DictionaryEncodedStringIndexSupplier(
factory,
GenericIndexed.fromIterable(Collections.singletonList("foo1"), GenericIndexed.STRING_STRATEGY),
GenericIndexed.fromIterable(Collections.singletonList(foo1BitMap), serdeFactory.getObjectStrategy()),
null
);
}
return null;
}
};
private static final ExtractionFn DIM_EXTRACTION_FN = new DimExtractionFn()
{
@ -227,7 +157,7 @@ public class ExtractionDimFilterTest
public void testEmpty()
{
Filter extractionFilter = new SelectorDimFilter("foo", "NFDJUKFNDSJFNS", DIM_EXTRACTION_FN).toFilter();
ImmutableBitmap immutableBitmap = extractionFilter.getBitmapIndex(BITMAP_INDEX_SELECTOR);
ImmutableBitmap immutableBitmap = Filters.computeDefaultBitmapResults(extractionFilter, BITMAP_INDEX_SELECTOR);
Assert.assertEquals(0, immutableBitmap.size());
}
@ -235,7 +165,7 @@ public class ExtractionDimFilterTest
public void testNull()
{
Filter extractionFilter = new SelectorDimFilter("FDHJSFFHDS", "extractDimVal", DIM_EXTRACTION_FN).toFilter();
ImmutableBitmap immutableBitmap = extractionFilter.getBitmapIndex(BITMAP_INDEX_SELECTOR);
ImmutableBitmap immutableBitmap = Filters.computeDefaultBitmapResults(extractionFilter, BITMAP_INDEX_SELECTOR);
Assert.assertEquals(0, immutableBitmap.size());
}
@ -243,7 +173,7 @@ public class ExtractionDimFilterTest
public void testNormal()
{
Filter extractionFilter = new SelectorDimFilter("foo", "extractDimVal", DIM_EXTRACTION_FN).toFilter();
ImmutableBitmap immutableBitmap = extractionFilter.getBitmapIndex(BITMAP_INDEX_SELECTOR);
ImmutableBitmap immutableBitmap = Filters.computeDefaultBitmapResults(extractionFilter, BITMAP_INDEX_SELECTOR);
Assert.assertEquals(1, immutableBitmap.size());
}
@ -252,23 +182,23 @@ public class ExtractionDimFilterTest
{
Assert.assertEquals(
1,
Filters
.toFilter(DimFilters.or(new ExtractionDimFilter("foo", "extractDimVal", DIM_EXTRACTION_FN, null)))
.getBitmapIndex(BITMAP_INDEX_SELECTOR)
.size()
Filters.computeDefaultBitmapResults(
Filters.toFilter(DimFilters.or(new ExtractionDimFilter("foo", "extractDimVal", DIM_EXTRACTION_FN, null))),
BITMAP_INDEX_SELECTOR
).size()
);
Assert.assertEquals(
1,
Filters
.toFilter(
Filters.computeDefaultBitmapResults(
Filters.toFilter(
DimFilters.or(
new ExtractionDimFilter("foo", "extractDimVal", DIM_EXTRACTION_FN, null),
new ExtractionDimFilter("foo", "DOES NOT EXIST", DIM_EXTRACTION_FN, null)
)
)
.getBitmapIndex(BITMAP_INDEX_SELECTOR)
.size()
),
BITMAP_INDEX_SELECTOR
).size()
);
}
@ -277,23 +207,23 @@ public class ExtractionDimFilterTest
{
Assert.assertEquals(
1,
Filters
.toFilter(DimFilters.or(new ExtractionDimFilter("foo", "extractDimVal", DIM_EXTRACTION_FN, null)))
.getBitmapIndex(BITMAP_INDEX_SELECTOR)
.size()
Filters.computeDefaultBitmapResults(
Filters.toFilter(DimFilters.or(new ExtractionDimFilter("foo", "extractDimVal", DIM_EXTRACTION_FN, null))),
BITMAP_INDEX_SELECTOR
).size()
);
Assert.assertEquals(
1,
Filters
.toFilter(
Filters.computeDefaultBitmapResults(
Filters.toFilter(
DimFilters.and(
new ExtractionDimFilter("foo", "extractDimVal", DIM_EXTRACTION_FN, null),
new ExtractionDimFilter("foo", "extractDimVal", DIM_EXTRACTION_FN, null)
)
)
.getBitmapIndex(BITMAP_INDEX_SELECTOR)
.size()
),
BITMAP_INDEX_SELECTOR
).size()
);
}
@ -303,18 +233,18 @@ public class ExtractionDimFilterTest
Assert.assertEquals(
1,
Filters
.toFilter(DimFilters.or(new ExtractionDimFilter("foo", "extractDimVal", DIM_EXTRACTION_FN, null)))
.getBitmapIndex(BITMAP_INDEX_SELECTOR)
.size()
Filters.computeDefaultBitmapResults(
Filters.toFilter(DimFilters.or(new ExtractionDimFilter("foo", "extractDimVal", DIM_EXTRACTION_FN, null))),
BITMAP_INDEX_SELECTOR
).size()
);
Assert.assertEquals(
1,
Filters
.toFilter(DimFilters.not(new ExtractionDimFilter("foo", "DOES NOT EXIST", DIM_EXTRACTION_FN, null)))
.getBitmapIndex(BITMAP_INDEX_SELECTOR)
.size()
Filters.computeDefaultBitmapResults(
Filters.toFilter(DimFilters.not(new ExtractionDimFilter("foo", "DOES NOT EXIST", DIM_EXTRACTION_FN, null))),
BITMAP_INDEX_SELECTOR
).size()
);
}
}

18
processing/src/test/java/org/apache/druid/segment/filter/FilterPartitionTest.java
View File

@ -29,7 +29,7 @@ import org.apache.druid.js.JavaScriptConfig;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.query.extraction.JavaScriptExtractionFn;
import org.apache.druid.query.filter.AndDimFilter;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.DimFilter;
import org.apache.druid.query.filter.DruidDoublePredicate;
import org.apache.druid.query.filter.DruidFloatPredicate;
@ -39,10 +39,11 @@ import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.FilterTuning;
import org.apache.druid.query.filter.OrDimFilter;
import org.apache.druid.query.filter.SelectorDimFilter;
import org.apache.druid.segment.ColumnSelectorBitmapIndexSelector;
import org.apache.druid.segment.ColumnSelectorColumnIndexSelector;
import org.apache.druid.segment.IndexBuilder;
import org.apache.druid.segment.QueryableIndexStorageAdapter;
import org.apache.druid.segment.StorageAdapter;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.filter.cnf.CNFFilterExplosionException;
import org.junit.AfterClass;
import org.junit.Assert;
@ -50,6 +51,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import javax.annotation.Nullable;
import java.io.Closeable;
import java.util.Arrays;
import java.util.List;
@ -77,10 +79,11 @@ public class FilterPartitionTest extends BaseFilterTest
super(dimension, value, filterTuning);
}
@Nullable
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
return false;
return null;
}
}
@ -95,10 +98,11 @@ public class FilterPartitionTest extends BaseFilterTest
super(dimension, predicateFactory, extractionFn);
}
@Nullable
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
return false;
return null;
}
}
@ -723,7 +727,7 @@ public class FilterPartitionTest extends BaseFilterTest
return;
}
QueryableIndexStorageAdapter storageAdapter = (QueryableIndexStorageAdapter) adapter;
final ColumnSelectorBitmapIndexSelector bitmapIndexSelector = storageAdapter.makeBitmapIndexSelector(BaseFilterTest.VIRTUAL_COLUMNS);
final ColumnSelectorColumnIndexSelector bitmapIndexSelector = storageAdapter.makeBitmapIndexSelector(BaseFilterTest.VIRTUAL_COLUMNS);
// has bitmap index, will use it by default
Filter normalFilter = new SelectorFilter("dim1", "HELLO");

147
processing/src/test/java/org/apache/druid/segment/filter/FiltersTest.java
View File

@ -1,147 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment.filter;
import com.google.common.collect.Lists;
import it.unimi.dsi.fastutil.ints.IntIterators;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ConciseBitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.collections.bitmap.MutableBitmap;
import org.apache.druid.segment.IntIteratorUtils;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.Assert;
import org.junit.Test;
import javax.annotation.Nullable;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;
public class FiltersTest extends InitializedNullHandlingTest
{
@Test
public void testEstimateSelectivityOfBitmapList()
{
final int bitmapNum = 100;
final List<ImmutableBitmap> bitmaps = Lists.newArrayListWithCapacity(bitmapNum);
final BitmapIndex bitmapIndex = makeNonOverlappedBitmapIndexes(bitmapNum, bitmaps);
final double estimated = Filters.estimateSelectivity(
bitmapIndex,
IntIteratorUtils.toIntList(IntIterators.fromTo(0, bitmapNum)),
10000
);
final double expected = 0.1;
Assert.assertEquals(expected, estimated, 0.00001);
}
private static BitmapIndex getBitmapIndex(final List<ImmutableBitmap> bitmapList)
{
return new BitmapIndex()
{
@Override
public int getCardinality()
{
return 10;
}
@Override
public String getValue(int index)
{
throw new UnsupportedOperationException();
}
@Override
public boolean hasNulls()
{
return false;
}
@Override
public BitmapFactory getBitmapFactory()
{
return new ConciseBitmapFactory();
}
@Override
public int getIndex(String value)
{
throw new UnsupportedOperationException();
}
@Override
public ImmutableBitmap getBitmapForValue(@Nullable String value)
{
throw new UnsupportedOperationException();
}
@Override
public Iterable<ImmutableBitmap> getBitmapsInRange(
@Nullable String startValue,
boolean startStrict,
@Nullable String endValue,
boolean endStrict
)
{
throw new UnsupportedOperationException();
}
@Override
public Iterable<ImmutableBitmap> getBitmapsInRange(
@Nullable String startValue,
boolean startStrict,
@Nullable String endValue,
boolean endStrict,
Predicate<String> matcher
)
{
throw new UnsupportedOperationException();
}
@Override
public Iterable<ImmutableBitmap> getBitmapsForValues(Set<String> values)
{
throw new UnsupportedOperationException();
}
@Override
public ImmutableBitmap getBitmap(int idx)
{
return bitmapList.get(idx);
}
};
}
private static BitmapIndex makeNonOverlappedBitmapIndexes(final int bitmapNum, final List<ImmutableBitmap> bitmaps)
{
final BitmapIndex bitmapIndex = getBitmapIndex(bitmaps);
final BitmapFactory factory = bitmapIndex.getBitmapFactory();
for (int i = 0; i < bitmapNum; i++) {
final MutableBitmap mutableBitmap = factory.makeEmptyMutableBitmap();
for (int j = 0; j < 10; j++) {
mutableBitmap.add(i * 10 + j);
}
bitmaps.add(factory.makeImmutableBitmap(mutableBitmap));
}
return bitmapIndex;
}
}

27
processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java
View File

@ -36,13 +36,12 @@ import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.guava.Sequence;
import org.apache.druid.js.JavaScriptConfig;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.Result;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.JavaScriptAggregatorFactory;
import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.DimFilters;
import org.apache.druid.query.filter.DruidDoublePredicate;
import org.apache.druid.query.filter.DruidFloatPredicate;
@ -63,6 +62,8 @@ import org.apache.druid.segment.Cursor;
import org.apache.druid.segment.DimensionSelector;
import org.apache.druid.segment.StorageAdapter;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.AllTrueBitmapColumnIndex;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.data.IndexedInts;
import org.apache.druid.segment.filter.Filters;
import org.apache.druid.segment.filter.SelectorFilter;
@ -75,6 +76,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import javax.annotation.Nullable;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
@ -657,14 +659,15 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
this.timestamp = timestamp;
}
@Nullable
@Override
public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory)
public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
{
return bitmapResultFactory.wrapAllTrue(Filters.allTrue(selector));
return new AllTrueBitmapColumnIndex(selector);
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
public double estimateSelectivity(ColumnIndexSelector indexSelector)
{
return 1;
}
@ -680,19 +683,7 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return true;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return true;
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, ColumnIndexSelector indexSelector)
{
return true;
}

5
processing/src/test/java/org/apache/druid/segment/serde/NullColumnPartSerdeTest.java
View File

@ -26,7 +26,6 @@ import org.apache.druid.segment.column.ColumnBuilder;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnConfig;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.Assert;
import org.junit.Test;
@ -41,7 +40,7 @@ public class NullColumnPartSerdeTest extends InitializedNullHandlingTest
{
final ObjectMapper mapper = new DefaultObjectMapper();
final NullColumnPartSerde partSerde = new NullColumnPartSerde(10, new RoaringBitmapSerdeFactory(null));
final NullColumnPartSerde partSerde = new NullColumnPartSerde(10);
final String json = mapper.writeValueAsString(partSerde);
Assert.assertEquals(partSerde, mapper.readValue(json, ColumnPartSerde.class));
}
@ -49,7 +48,7 @@ public class NullColumnPartSerdeTest extends InitializedNullHandlingTest
@Test
public void testDeserializer()
{
final NullColumnPartSerde partSerde = new NullColumnPartSerde(10, new RoaringBitmapSerdeFactory(null));
final NullColumnPartSerde partSerde = new NullColumnPartSerde(10);
final ColumnBuilder builder = new ColumnBuilder().setType(ValueType.DOUBLE);
partSerde.getDeserializer().read(Mockito.mock(ByteBuffer.class), builder, Mockito.mock(ColumnConfig.class));
final ColumnCapabilities columnCapabilities = builder.build().getCapabilities();

35
processing/src/test/java/org/apache/druid/segment/virtual/DummyStringVirtualColumn.java
View File

@ -33,10 +33,10 @@ import org.apache.druid.segment.IdLookup;
import org.apache.druid.segment.NilColumnValueSelector;
import org.apache.druid.segment.VirtualColumn;
import org.apache.druid.segment.column.BaseColumn;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.StringDictionaryEncodedColumn;
import org.apache.druid.segment.data.IndexedInts;
@ -165,19 +165,32 @@ public class DummyStringVirtualColumn implements VirtualColumn
}
}
@Nullable
@Override
public @Nullable BitmapIndex getBitmapIndex(String columnName, ColumnSelector columnSelector)
public ColumnIndexSupplier getIndexSupplier(
String columnName,
ColumnSelector columnSelector
)
{
if (enableBitmaps) {
ColumnHolder holder = columnSelector.getColumnHolder(baseColumnName);
if (holder == null) {
return null;
}
return new ColumnIndexSupplier()
{
return holder.getBitmapIndex();
} else {
throw new UnsupportedOperationException("not supported");
}
@Nullable
@Override
public <T> T as(Class<T> clazz)
{
if (enableBitmaps) {
ColumnHolder holder = columnSelector.getColumnHolder(baseColumnName);
if (holder == null) {
return null;
}
return holder.getIndexSupplier().as(clazz);
} else {
return null;
}
}
};
}
@Override

45
processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnSelectorTest.java
View File

@ -28,17 +28,18 @@ import org.apache.druid.collections.bitmap.RoaringBitmapFactory;
import org.apache.druid.data.input.MapBasedRow;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorBitmapIndexSelector;
import org.apache.druid.segment.ColumnSelectorColumnIndexSelector;
import org.apache.druid.segment.ColumnValueSelector;
import org.apache.druid.segment.DimensionSelector;
import org.apache.druid.segment.RowAdapters;
import org.apache.druid.segment.RowBasedColumnSelectorFactory;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.filter.SelectorFilter;
@ -168,9 +169,10 @@ public class ListFilteredVirtualColumnSelectorTest extends InitializedNullHandli
ColumnSelector selector = EasyMock.createMock(ColumnSelector.class);
ColumnHolder holder = EasyMock.createMock(ColumnHolder.class);
BitmapIndex index = EasyMock.createMock(BitmapIndex.class);
DictionaryEncodedStringValueIndex index = EasyMock.createMock(DictionaryEncodedStringValueIndex.class);
ImmutableBitmap bitmap = EasyMock.createMock(ImmutableBitmap.class);
BitmapFactory bitmapFactory = EasyMock.createMock(BitmapFactory.class);
ColumnIndexSupplier indexSupplier = EasyMock.createMock(ColumnIndexSupplier.class);
EasyMock.expect(selector.getColumnHolder(COLUMN_NAME)).andReturn(holder).atLeastOnce();
EasyMock.expect(selector.getColumnCapabilities(COLUMN_NAME))
@ -181,7 +183,9 @@ public class ListFilteredVirtualColumnSelectorTest extends InitializedNullHandli
.setHasBitmapIndexes(true)
).anyTimes();
EasyMock.expect(holder.getBitmapIndex()).andReturn(index).atLeastOnce();
EasyMock.expect(holder.getIndexSupplier()).andReturn(indexSupplier).atLeastOnce();
EasyMock.expect(indexSupplier.as(DictionaryEncodedStringValueIndex.class)).andReturn(index).atLeastOnce();
EasyMock.expect(index.getCardinality()).andReturn(3).atLeastOnce();
EasyMock.expect(index.getValue(0)).andReturn("a").atLeastOnce();
@ -189,21 +193,21 @@ public class ListFilteredVirtualColumnSelectorTest extends InitializedNullHandli
EasyMock.expect(index.getValue(2)).andReturn("c").atLeastOnce();
EasyMock.expect(index.getBitmap(2)).andReturn(bitmap).once();
EasyMock.expect(index.getBitmapFactory()).andReturn(bitmapFactory).once();
EasyMock.expect(index.hasNulls()).andReturn(true).once();
EasyMock.replay(selector, holder, index, bitmap, bitmapFactory);
EasyMock.replay(selector, holder, indexSupplier, index, bitmap, bitmapFactory);
ColumnSelectorBitmapIndexSelector bitmapIndexSelector = new ColumnSelectorBitmapIndexSelector(
ColumnSelectorColumnIndexSelector bitmapIndexSelector = new ColumnSelectorColumnIndexSelector(
new RoaringBitmapFactory(),
VirtualColumns.create(Collections.singletonList(virtualColumn)),
selector
);
SelectorFilter filter = new SelectorFilter(ALLOW_VIRTUAL_NAME, "a");
Assert.assertTrue(filter.shouldUseBitmapIndex(bitmapIndexSelector));
Assert.assertNotNull(filter.getBitmapColumnIndex(bitmapIndexSelector));
BitmapIndex listFilteredIndex = bitmapIndexSelector.getBitmapIndex(ALLOW_VIRTUAL_NAME);
DictionaryEncodedStringValueIndex listFilteredIndex =
bitmapIndexSelector.getIndexSupplier(ALLOW_VIRTUAL_NAME).as(DictionaryEncodedStringValueIndex.class);
Assert.assertEquals(-1, listFilteredIndex.getIndex("a"));
Assert.assertEquals(0, listFilteredIndex.getIndex("b"));
Assert.assertEquals(1, listFilteredIndex.getIndex("c"));
@ -211,10 +215,9 @@ public class ListFilteredVirtualColumnSelectorTest extends InitializedNullHandli
Assert.assertEquals("b", listFilteredIndex.getValue(0));
Assert.assertEquals("c", listFilteredIndex.getValue(1));
Assert.assertEquals(bitmap, listFilteredIndex.getBitmap(1));
Assert.assertEquals(bitmapFactory, listFilteredIndex.getBitmapFactory());
Assert.assertTrue(listFilteredIndex.hasNulls());
EasyMock.verify(selector, holder, index, bitmap, bitmapFactory);
EasyMock.verify(selector, holder, indexSupplier, index, bitmap, bitmapFactory);
}
@Test
@ -230,8 +233,9 @@ public class ListFilteredVirtualColumnSelectorTest extends InitializedNullHandli
ColumnSelector selector = EasyMock.createMock(ColumnSelector.class);
ColumnHolder holder = EasyMock.createMock(ColumnHolder.class);
BitmapIndex index = EasyMock.createMock(BitmapIndex.class);
DictionaryEncodedStringValueIndex index = EasyMock.createMock(DictionaryEncodedStringValueIndex.class);
ImmutableBitmap bitmap = EasyMock.createMock(ImmutableBitmap.class);
ColumnIndexSupplier indexSupplier = EasyMock.createMock(ColumnIndexSupplier.class);
BitmapFactory bitmapFactory = EasyMock.createMock(BitmapFactory.class);
EasyMock.expect(selector.getColumnHolder(COLUMN_NAME)).andReturn(holder).atLeastOnce();
@ -242,38 +246,37 @@ public class ListFilteredVirtualColumnSelectorTest extends InitializedNullHandli
.setDictionaryValuesSorted(true)
.setHasBitmapIndexes(true)
).anyTimes();
EasyMock.expect(holder.getBitmapIndex()).andReturn(index).atLeastOnce();
EasyMock.expect(holder.getIndexSupplier()).andReturn(indexSupplier).atLeastOnce();
EasyMock.expect(indexSupplier.as(DictionaryEncodedStringValueIndex.class)).andReturn(index).atLeastOnce();
EasyMock.expect(index.getCardinality()).andReturn(3).atLeastOnce();
EasyMock.expect(index.getValue(0)).andReturn("a").atLeastOnce();
EasyMock.expect(index.getValue(1)).andReturn("b").atLeastOnce();
EasyMock.expect(index.getValue(2)).andReturn("c").atLeastOnce();
EasyMock.expect(index.getBitmap(0)).andReturn(bitmap).once();
EasyMock.expect(index.getBitmapFactory()).andReturn(bitmapFactory).once();
EasyMock.expect(index.hasNulls()).andReturn(true).once();
EasyMock.replay(selector, holder, index, bitmap, bitmapFactory);
EasyMock.replay(selector, holder, indexSupplier, index, bitmap, bitmapFactory);
ColumnSelectorBitmapIndexSelector bitmapIndexSelector = new ColumnSelectorBitmapIndexSelector(
ColumnSelectorColumnIndexSelector bitmapIndexSelector = new ColumnSelectorColumnIndexSelector(
new RoaringBitmapFactory(),
VirtualColumns.create(Collections.singletonList(virtualColumn)),
selector
);
SelectorFilter filter = new SelectorFilter(DENY_VIRTUAL_NAME, "c");
Assert.assertTrue(filter.shouldUseBitmapIndex(bitmapIndexSelector));
Assert.assertNotNull(filter.getBitmapColumnIndex(bitmapIndexSelector));
BitmapIndex listFilteredIndex = bitmapIndexSelector.getBitmapIndex(DENY_VIRTUAL_NAME);
DictionaryEncodedStringValueIndex listFilteredIndex =
bitmapIndexSelector.getIndexSupplier(DENY_VIRTUAL_NAME).as(DictionaryEncodedStringValueIndex.class);
Assert.assertEquals(-1, listFilteredIndex.getIndex("a"));
Assert.assertEquals(-1, listFilteredIndex.getIndex("b"));
Assert.assertEquals(0, listFilteredIndex.getIndex("c"));
Assert.assertEquals(1, listFilteredIndex.getCardinality());
Assert.assertEquals(bitmap, listFilteredIndex.getBitmap(1));
Assert.assertEquals(bitmapFactory, listFilteredIndex.getBitmapFactory());
Assert.assertTrue(listFilteredIndex.hasNulls());
EasyMock.verify(selector, holder, index, bitmap, bitmapFactory);
EasyMock.verify(selector, holder, indexSupplier, index, bitmap, bitmapFactory);
}
private void assertCapabilities(VirtualizedColumnSelectorFactory selectorFactory, String columnName)

75
services/src/main/java/org/apache/druid/cli/DumpSegment.java
View File

@ -74,9 +74,10 @@ import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.QueryableIndexSegment;
import org.apache.druid.segment.QueryableIndexStorageAdapter;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.ColumnConfig;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex;
import org.apache.druid.segment.data.BitmapSerdeFactory;
import org.apache.druid.segment.data.ConciseBitmapSerdeFactory;
import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
@ -182,7 +183,7 @@ public class DumpSegment extends GuiceRunnable
runMetadata(injector, index);
break;
case BITMAPS:
runBitmaps(injector, index);
runBitmaps(injector, outputFileName, index, getColumnsToInclude(index), decompressBitmaps);
break;
default:
throw new ISE("dumpType[%s] has no handler", dumpType);
@ -313,7 +314,14 @@ public class DumpSegment extends GuiceRunnable
);
}
private void runBitmaps(final Injector injector, final QueryableIndex index) throws IOException
@VisibleForTesting
public static void runBitmaps(
final Injector injector,
final String outputFileName,
final QueryableIndex index,
List<String> columnNames,
boolean decompressBitmaps
) throws IOException
{
final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
@ -330,35 +338,33 @@ public class DumpSegment extends GuiceRunnable
);
}
final List<String> columnNames = getColumnsToInclude(index);
withOutputStream(
new Function<OutputStream, Object>()
{
@Override
public Object apply(final OutputStream out)
{
try (final JsonGenerator jg = objectMapper.getFactory().createGenerator(out)) {
out -> {
try (final JsonGenerator jg = objectMapper.getFactory().createGenerator(out)) {
jg.writeStartObject();
{
jg.writeObjectField("bitmapSerdeFactory", bitmapSerdeFactory);
jg.writeFieldName("bitmaps");
jg.writeStartObject();
{
jg.writeObjectField("bitmapSerdeFactory", bitmapSerdeFactory);
jg.writeFieldName("bitmaps");
jg.writeStartObject();
{
for (final String columnName : columnNames) {
final ColumnHolder columnHolder = index.getColumnHolder(columnName);
final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
if (bitmapIndex == null) {
for (final String columnName : columnNames) {
final ColumnHolder columnHolder = index.getColumnHolder(columnName);
final ColumnIndexSupplier indexSupplier = columnHolder.getIndexSupplier();
if (indexSupplier == null) {
jg.writeNullField(columnName);
} else {
DictionaryEncodedStringValueIndex valueIndex =
indexSupplier.as(DictionaryEncodedStringValueIndex.class);
if (valueIndex == null) {
jg.writeNullField(columnName);
} else {
jg.writeFieldName(columnName);
jg.writeStartObject();
for (int i = 0; i < bitmapIndex.getCardinality(); i++) {
String val = bitmapIndex.getValue(i);
for (int i = 0; i < valueIndex.getCardinality(); i++) {
String val = valueIndex.getValue(i);
// respect nulls if they are present in the dictionary
jg.writeFieldName(val == null ? "null" : val);
final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
final ImmutableBitmap bitmap = valueIndex.getBitmap(i);
if (decompressBitmaps) {
jg.writeStartArray();
final IntIterator iterator = bitmap.iterator();
@ -378,17 +384,18 @@ public class DumpSegment extends GuiceRunnable
}
}
}
jg.writeEndObject();
}
jg.writeEndObject();
}
catch (IOException e) {
throw new RuntimeException(e);
}
return null;
jg.writeEndObject();
}
}
catch (IOException e) {
throw new RuntimeException(e);
}
return null;
},
outputFileName
);
}
@ -412,8 +419,13 @@ public class DumpSegment extends GuiceRunnable
return ImmutableList.copyOf(columnNames);
}
@SuppressForbidden(reason = "System#out")
private <T> T withOutputStream(Function<OutputStream, T> f) throws IOException
{
return withOutputStream(f, outputFileName);
}
@SuppressForbidden(reason = "System#out")
private static <T> T withOutputStream(Function<OutputStream, T> f, String outputFileName) throws IOException
{
if (outputFileName == null) {
return f.apply(System.out);
@ -489,5 +501,4 @@ public class DumpSegment extends GuiceRunnable
{
sequence.accumulate(null, (accumulated, in) -> null);
}
}

48
services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java
View File

@ -19,8 +19,15 @@
package org.apache.druid.cli;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import com.google.inject.Injector;
import com.google.inject.Key;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.collections.bitmap.RoaringBitmapFactory;
import org.apache.druid.guice.annotations.Json;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.java.util.common.guava.Sequence;
import org.apache.druid.java.util.common.guava.Sequences;
import org.apache.druid.query.DirectQueryProcessingPool;
@ -28,11 +35,16 @@ import org.apache.druid.query.Query;
import org.apache.druid.query.QueryRunner;
import org.apache.druid.query.QueryRunnerFactory;
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentMatchers;
import org.mockito.Mockito;
import java.io.IOException;
import java.util.Collections;
public class DumpSegmentTest
@ -55,4 +67,40 @@ public class DumpSegmentTest
Sequence actual = DumpSegment.executeQuery(injector, null, query);
Assert.assertSame(expected, actual);
}
@Test
public void testDumpBitmap() throws IOException
{
Injector injector = Mockito.mock(Injector.class);
QueryableIndex queryableIndex = Mockito.mock(QueryableIndex.class);
ObjectMapper mapper = new DefaultObjectMapper();
BitmapFactory bitmapFactory = new RoaringBitmapFactory();
ColumnHolder xHolder = Mockito.mock(ColumnHolder.class);
ColumnHolder yHolder = Mockito.mock(ColumnHolder.class);
ColumnIndexSupplier indexSupplier = Mockito.mock(ColumnIndexSupplier.class);
DictionaryEncodedStringValueIndex valueIndex = Mockito.mock(DictionaryEncodedStringValueIndex.class);
ImmutableBitmap bitmap = bitmapFactory.complement(bitmapFactory.makeEmptyImmutableBitmap(), 10);
Mockito.when(injector.getInstance(Key.get(ObjectMapper.class, Json.class))).thenReturn(mapper);
Mockito.when(queryableIndex.getBitmapFactoryForDimensions()).thenReturn(bitmapFactory);
Mockito.when(queryableIndex.getColumnHolder("x")).thenReturn(xHolder);
Mockito.when(queryableIndex.getColumnHolder("y")).thenReturn(yHolder);
Mockito.when(xHolder.getIndexSupplier()).thenReturn(indexSupplier);
Mockito.when(indexSupplier.as(DictionaryEncodedStringValueIndex.class)).thenReturn(valueIndex);
Mockito.when(valueIndex.getCardinality()).thenReturn(1);
Mockito.when(valueIndex.getBitmap(0)).thenReturn(bitmap);
Mockito.when(valueIndex.getValue(0)).thenReturn("val");
DumpSegment.runBitmaps(
injector,
null,
queryableIndex,
ImmutableList.of("x", "y"),
false
);
Assert.assertTrue(true);
}
}

2
sql/src/main/java/org/apache/druid/sql/calcite/run/NativeQueryMaker.java
View File

@ -140,7 +140,7 @@ public class NativeQueryMaker implements QueryMaker
// a BoundFilter is created internally for each of the values
// whereas when Vi s are String the Filters are converted as BoundFilter to SelectorFilter to InFilter
// which takes lesser processing for bitmaps
// So in a case where user executes a query with multiple numeric INs, flame graph shows BoundFilter.getBitmapResult
// So in a case where user executes a query with multiple numeric INs, flame graph shows BoundFilter.getBitmapColumnIndex
// and BoundFilter.match predicate eating up processing time which stalls a historical for a query with large number
// of numeric INs (> 10K). In such cases user should change the query to specify the IN clauses as String
// Instead of IN(v1,v2,v3) user should specify IN('v1','v2','v3')