Apache/druid
1
0
Fork 0
mirror of https://github.com/apache/druid.git synced 2025-02-22 18:30:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

add mechanism to control filter optimization in historical query processing (#8209)

Browse Source 
* add support for mechanism to control filter optimization in historical query processing

* oops

* adjust

* woo

* javadoc

* review comments

* fix

* default

* oops

* oof

* this will fix it

* more nullable, refactor DimFilter.getRequiredColumns to use Set, formatting

* extract class DimFilterToStringBuilder with common code from custom DimFilter toString implementations

* adjust variable naming

* missing nullable

* more nullable

* fix javadocs

* nullable

* address review comments

* javadocs, precondition

* nullable

* rename method to be consistent

* review comments

* remove tuning from ColumnComparisonFilter/ColumnComparisonDimFilter
This commit is contained in:
Clint Wylie 2019-08-09 16:36:18 -07:00 committed by GitHub
parent b28e252d9a
commit 1054d85171
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
75 changed files with 1370 additions and 615 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java
View File

@ -71,8 +71,8 @@ public class MomentSketchAggregatorFactory extends AggregatorFactory
public MomentSketchAggregatorFactory(
@JsonProperty("name") final String name,
@JsonProperty("fieldName") final String fieldName,
@Nullable @JsonProperty("k") final Integer k,
@Nullable @JsonProperty("compress") final Boolean compress
@JsonProperty("k") @Nullable final Integer k,
@JsonProperty("compress") @Nullable final Boolean compress
)
{
this(name, fieldName, k, compress, AggregatorUtil.MOMENTS_SKETCH_BUILD_CACHE_TYPE_ID);

2
extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java
View File

@ -76,7 +76,7 @@ public class TDigestSketchAggregatorFactory extends AggregatorFactory
public TDigestSketchAggregatorFactory(
@JsonProperty("name") final String name,
@JsonProperty("fieldName") final String fieldName,
@Nullable @JsonProperty("compression") final Integer compression
@JsonProperty("compression") @Nullable final Integer compression
)
{
this(name, fieldName, compression, AggregatorUtil.TDIGEST_BUILD_SKETCH_CACHE_TYPE_ID);

8
extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java
View File

@ -39,10 +39,10 @@ public class SketchMergeAggregatorFactory extends SketchAggregatorFactory
public SketchMergeAggregatorFactory(
@JsonProperty("name") String name,
@JsonProperty("fieldName") String fieldName,
@Nullable @JsonProperty("size") Integer size,
@Nullable @JsonProperty("shouldFinalize") Boolean shouldFinalize,
@Nullable @JsonProperty("isInputThetaSketch") Boolean isInputThetaSketch,
@Nullable @JsonProperty("errorBoundsStdDev") Integer errorBoundsStdDev
@JsonProperty("size") @Nullable Integer size,
@JsonProperty("shouldFinalize") @Nullable Boolean shouldFinalize,
@JsonProperty("isInputThetaSketch") @Nullable Boolean isInputThetaSketch,
@JsonProperty("errorBoundsStdDev") @Nullable Integer errorBoundsStdDev
)
{
super(name, fieldName, size, AggregatorUtil.SKETCH_MERGE_CACHE_TYPE_ID);

2
extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java
View File

@ -71,7 +71,7 @@ public class BloomFilterAggregatorFactory extends AggregatorFactory
public BloomFilterAggregatorFactory(
@JsonProperty("name") String name,
@JsonProperty("field") final DimensionSpec field,
@Nullable @JsonProperty("maxNumEntries") Integer maxNumEntries
@JsonProperty("maxNumEntries") @Nullable Integer maxNumEntries
)
{
this.name = name;

85
extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java
View File

@ -20,18 +20,21 @@
package org.apache.druid.query.filter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import com.google.common.hash.HashCode;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.segment.filter.DimensionPredicateFilter;
import java.util.HashSet;
import javax.annotation.Nullable;
import java.util.Objects;
import java.util.Set;
/**
*/
@ -41,13 +44,17 @@ public class BloomDimFilter implements DimFilter
private final String dimension;
private final BloomKFilter bloomKFilter;
private final HashCode hash;
@Nullable
private final ExtractionFn extractionFn;
@Nullable
private final FilterTuning filterTuning;
@JsonCreator
public BloomDimFilter(
@JsonProperty("dimension") String dimension,
@JsonProperty("bloomKFilter") BloomKFilterHolder bloomKFilterHolder,
@JsonProperty("extractionFn") ExtractionFn extractionFn
@JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn,
@JsonProperty("filterTuning") @Nullable FilterTuning filterTuning
)
{
Preconditions.checkArgument(dimension != null, "dimension must not be null");
@ -56,6 +63,13 @@ public class BloomDimFilter implements DimFilter
this.bloomKFilter = bloomKFilterHolder.getFilter();
this.hash = bloomKFilterHolder.getFilterHash();
this.extractionFn = extractionFn;
this.filterTuning = filterTuning;
}
@VisibleForTesting
public BloomDimFilter(String dimension, BloomKFilterHolder bloomKFilterHolder, @Nullable ExtractionFn extractionFn)
{
this(dimension, bloomKFilterHolder, extractionFn, null);
}
@Override
@ -152,7 +166,8 @@ public class BloomDimFilter implements DimFilter
};
}
},
extractionFn
extractionFn,
filterTuning
);
}
@ -168,20 +183,40 @@ public class BloomDimFilter implements DimFilter
return bloomKFilter;
}
@Nullable
@JsonProperty
public ExtractionFn getExtractionFn()
{
return extractionFn;
}
@Nullable
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonProperty
public FilterTuning getFilterTuning()
{
return filterTuning;
}
@Override
public RangeSet<String> getDimensionRangeSet(String dimension)
{
return null;
}
@Override
public Set<String> getRequiredColumns()
{
return ImmutableSet.of(dimension);
}
@Override
public String toString()
{
if (extractionFn != null) {
return StringUtils.format("%s(%s) = %s", extractionFn, dimension, hash.toString());
} else {
return StringUtils.format("%s = %s", dimension, hash.toString());
}
return new DimFilterToStringBuilder().appendDimension(dimension, extractionFn)
.appendEquals(hash.toString())
.appendFilterTuning(filterTuning)
.build();
}
@Override
@ -193,36 +228,16 @@ public class BloomDimFilter implements DimFilter
if (o == null || getClass() != o.getClass()) {
return false;
}
BloomDimFilter that = (BloomDimFilter) o;
if (!dimension.equals(that.dimension)) {
return false;
}
if (hash != null ? !hash.equals(that.hash) : that.hash != null) {
return false;
}
return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null;
}
@Override
public RangeSet<String> getDimensionRangeSet(String dimension)
{
return null;
}
@Override
public HashSet<String> getRequiredColumns()
{
return Sets.newHashSet(dimension);
return dimension.equals(that.dimension) &&
hash.equals(that.hash) &&
Objects.equals(extractionFn, that.extractionFn) &&
Objects.equals(filterTuning, that.filterTuning);
}
@Override
public int hashCode()
{
int result = dimension.hashCode();
result = 31 * result + (hash != null ? hash.hashCode() : 0);
result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0);
return result;
return Objects.hash(dimension, hash, extractionFn, filterTuning);
}
}

4
extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/sql/BloomFilterOperatorConversion.java
View File

@ -100,7 +100,8 @@ public class BloomFilterOperatorConversion extends DirectOperatorConversion
return new BloomDimFilter(
druidExpression.getSimpleExtraction().getColumn(),
holder,
druidExpression.getSimpleExtraction().getExtractionFn()
druidExpression.getSimpleExtraction().getExtractionFn(),
null
);
} else if (virtualColumnRegistry != null) {
VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression(
@ -114,6 +115,7 @@ public class BloomFilterOperatorConversion extends DirectOperatorConversion
return new BloomDimFilter(
virtualColumn.getOutputName(),
holder,
null,
null
);
} else {

2
extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java
View File

@ -65,7 +65,7 @@ public class ApproximateHistogramAggregatorFactory extends AggregatorFactory
@JsonProperty("numBuckets") Integer numBuckets,
@JsonProperty("lowerLimit") Float lowerLimit,
@JsonProperty("upperLimit") Float upperLimit,
@Nullable @JsonProperty("finalizeAsBase64Binary") Boolean finalizeAsBase64Binary
@JsonProperty("finalizeAsBase64Binary") @Nullable Boolean finalizeAsBase64Binary
)
{

2
extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java
View File

@ -46,7 +46,7 @@ public class ApproximateHistogramFoldingAggregatorFactory extends ApproximateHis
@JsonProperty("numBuckets") Integer numBuckets,
@JsonProperty("lowerLimit") Float lowerLimit,
@JsonProperty("upperLimit") Float upperLimit,
@Nullable @JsonProperty("finalizeAsBase64Binary") Boolean finalizeAsBase64Binary
@JsonProperty("finalizeAsBase64Binary") @Nullable Boolean finalizeAsBase64Binary
)
{
super(name, fieldName, resolution, numBuckets, lowerLimit, upperLimit, finalizeAsBase64Binary);

4
extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java
View File

@ -59,11 +59,11 @@ public class FixedBucketsHistogramAggregatorFactory extends AggregatorFactory
public FixedBucketsHistogramAggregatorFactory(
@JsonProperty("name") String name,
@JsonProperty("fieldName") String fieldName,
@Nullable @JsonProperty("numBuckets") Integer numBuckets,
@JsonProperty("numBuckets") @Nullable Integer numBuckets,
@JsonProperty("lowerLimit") double lowerLimit,
@JsonProperty("upperLimit") double upperLimit,
@JsonProperty("outlierHandlingMode") FixedBucketsHistogram.OutlierHandlingMode outlierHandlingMode,
@Nullable @JsonProperty("finalizeAsBase64Binary") Boolean finalizeAsBase64Binary
@JsonProperty("finalizeAsBase64Binary") @Nullable Boolean finalizeAsBase64Binary
)
{
this.name = name;

6
extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/JdbcExtractionNamespace.java
View File

@ -59,9 +59,9 @@ public class JdbcExtractionNamespace implements ExtractionNamespace
@NotNull @JsonProperty(value = "table", required = true) final String table,
@NotNull @JsonProperty(value = "keyColumn", required = true) final String keyColumn,
@NotNull @JsonProperty(value = "valueColumn", required = true) final String valueColumn,
@Nullable @JsonProperty(value = "tsColumn", required = false) final String tsColumn,
@Nullable @JsonProperty(value = "filter", required = false) final String filter,
@Min(0) @Nullable @JsonProperty(value = "pollPeriod", required = false) final Period pollPeriod
@JsonProperty(value = "tsColumn", required = false) @Nullable final String tsColumn,
@JsonProperty(value = "filter", required = false) @Nullable final String filter,
@Min(0) @JsonProperty(value = "pollPeriod", required = false) @Nullable final Period pollPeriod
)
{
this.connectorConfig = Preconditions.checkNotNull(connectorConfig, "connectorConfig");

2
extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/UriExtractionNamespace.java
View File

@ -85,7 +85,7 @@ public class UriExtractionNamespace implements ExtractionNamespace
String fileRegex,
@JsonProperty(value = "namespaceParseSpec", required = true)
FlatDataParser namespaceParseSpec,
@Min(0) @Nullable @JsonProperty(value = "pollPeriod", required = false)
@Min(0) @JsonProperty(value = "pollPeriod", required = false) @Nullable
Period pollPeriod,
@Deprecated
@JsonProperty(value = "versionRegex", required = false)

2
extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcHadoopInputRowParser.java
View File

@ -44,7 +44,7 @@ public class OrcHadoopInputRowParser implements InputRowParser<OrcStruct>
@JsonCreator
public OrcHadoopInputRowParser(
@JsonProperty("parseSpec") ParseSpec parseSpec,
@Nullable @JsonProperty("binaryAsString") Boolean binaryAsString
@JsonProperty("binaryAsString") @Nullable Boolean binaryAsString
)
{
this.parseSpec = parseSpec;

18
indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java
View File

@ -156,15 +156,15 @@ public class CompactionTask extends AbstractBatchIndexTask
@JsonProperty("id") final String id,
@JsonProperty("resource") final TaskResource taskResource,
@JsonProperty("dataSource") final String dataSource,
@Nullable @JsonProperty("interval") final Interval interval,
@Nullable @JsonProperty("segments") final List<DataSegment> segments,
@Nullable @JsonProperty("dimensions") final DimensionsSpec dimensions,
@Nullable @JsonProperty("dimensionsSpec") final DimensionsSpec dimensionsSpec,
@Nullable @JsonProperty("metricsSpec") final AggregatorFactory[] metricsSpec,
@Nullable @JsonProperty("segmentGranularity") final Granularity segmentGranularity,
@Nullable @JsonProperty("targetCompactionSizeBytes") final Long targetCompactionSizeBytes,
@Nullable @JsonProperty("tuningConfig") final IndexTuningConfig tuningConfig,
@Nullable @JsonProperty("context") final Map<String, Object> context,
@JsonProperty("interval") @Nullable final Interval interval,
@JsonProperty("segments") @Nullable final List<DataSegment> segments,
@JsonProperty("dimensions") @Nullable final DimensionsSpec dimensions,
@JsonProperty("dimensionsSpec") @Nullable final DimensionsSpec dimensionsSpec,
@JsonProperty("metricsSpec") @Nullable final AggregatorFactory[] metricsSpec,
@JsonProperty("segmentGranularity") @Nullable final Granularity segmentGranularity,
@JsonProperty("targetCompactionSizeBytes") @Nullable final Long targetCompactionSizeBytes,
@JsonProperty("tuningConfig") @Nullable final IndexTuningConfig tuningConfig,
@JsonProperty("context") @Nullable final Map<String, Object> context,
@JacksonInject ObjectMapper jsonMapper,
@JacksonInject AuthorizerMapper authorizerMapper,
@JacksonInject ChatHandlerProvider chatHandlerProvider,

4
indexing-service/src/main/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactory.java
View File

@ -101,10 +101,10 @@ public class IngestSegmentFirehoseFactory implements FiniteFirehoseFactory<Input
@JsonCreator
public IngestSegmentFirehoseFactory(
@JsonProperty("dataSource") final String dataSource,
@Nullable @JsonProperty("interval") Interval interval,
@JsonProperty("interval") @Nullable Interval interval,
// Specifying "segments" is intended only for when this FirehoseFactory has split itself,
// not for direct end user use.
@Nullable @JsonProperty("segments") List<WindowedSegmentId> segmentIds,
@JsonProperty("segments") @Nullable List<WindowedSegmentId> segmentIds,
@JsonProperty("filter") DimFilter dimFilter,
@JsonProperty("dimensions") List<String> dimensions,
@JsonProperty("metrics") List<String> metrics,

2
indexing-service/src/main/java/org/apache/druid/indexing/overlord/ImmutableWorkerInfo.java
View File

@ -51,7 +51,7 @@ public class ImmutableWorkerInfo
@JsonProperty("availabilityGroups") Set<String> availabilityGroups,
@JsonProperty("runningTasks") Collection<String> runningTasks,
@JsonProperty("lastCompletedTaskTime") DateTime lastCompletedTaskTime,
@Nullable @JsonProperty("blacklistedUntil") DateTime blacklistedUntil
@JsonProperty("blacklistedUntil") @Nullable DateTime blacklistedUntil
)
{
this.worker = worker;

4
processing/src/main/java/org/apache/druid/query/Druids.java
View File

@ -204,7 +204,7 @@ public class Druids
public TimeseriesQueryBuilder filters(String dimensionName, String value, String... values)
{
dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values), null);
dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values), null, null);
return this;
}
@ -361,7 +361,7 @@ public class Druids
public SearchQueryBuilder filters(String dimensionName, String value)
{
dimFilter = new SelectorDimFilter(dimensionName, value, null);
dimFilter = new SelectorDimFilter(dimensionName, value, null, null);
return this;
}

5
processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java
View File

@ -63,7 +63,7 @@ public class FilteredAggregatorFactory extends AggregatorFactory
public FilteredAggregatorFactory(
@JsonProperty("aggregator") AggregatorFactory delegate,
@JsonProperty("filter") DimFilter dimFilter,
@Nullable @JsonProperty("name") String name
@JsonProperty("name") @Nullable String name
)
{
Preconditions.checkNotNull(delegate, "aggregator");
@ -243,7 +243,8 @@ public class FilteredAggregatorFactory extends AggregatorFactory
new IntervalDimFilter(
intervalDimFilter.getDimension(),
effectiveFilterIntervals,
intervalDimFilter.getExtractionFn()
intervalDimFilter.getExtractionFn(),
intervalDimFilter.getFilterTuning()
),
this.name
);

4
processing/src/main/java/org/apache/druid/query/extraction/BucketExtractionFn.java
View File

@ -34,8 +34,8 @@ public class BucketExtractionFn implements ExtractionFn
@JsonCreator
public BucketExtractionFn(
@Nullable @JsonProperty("size") Double size,
@Nullable @JsonProperty("offset") Double offset
@JsonProperty("size") @Nullable Double size,
@JsonProperty("offset") @Nullable Double offset
)
{
this.size = size == null ? 1 : size;

3
processing/src/main/java/org/apache/druid/query/filter/AndDimFilter.java
View File

@ -32,6 +32,7 @@ import org.apache.druid.segment.filter.Filters;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/**
*/
@ -99,7 +100,7 @@ public class AndDimFilter implements DimFilter
}
@Override
public HashSet<String> getRequiredColumns()
public Set<String> getRequiredColumns()
{
HashSet<String> requiredColumns = new HashSet<>();
fields.forEach(field -> requiredColumns.addAll(field.getRequiredColumns()));

1
processing/src/main/java/org/apache/druid/query/filter/BitmapIndexSelector.java
View File

@ -38,6 +38,7 @@ public interface BitmapIndexSelector
boolean hasMultipleValues(String dimension);
int getNumRows();
BitmapFactory getBitmapFactory();
@Nullable
BitmapIndex getBitmapIndex(String dimension);
@Nullable
ImmutableBitmap getBitmapIndex(String dimension, String value);

46
processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java
View File

@ -19,9 +19,12 @@
package org.apache.druid.query.filter;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
public interface BooleanFilter extends Filter
{
@ -48,4 +51,47 @@ public interface BooleanFilter extends Filter
ColumnSelectorFactory columnSelectorFactory,
RowOffsetMatcherFactory rowOffsetMatcherFactory
);
@Override
default Set<String> getRequiredColumns()
{
Set<String> allColumns = new HashSet<>();
for (Filter f : getFilters()) {
allColumns.addAll(f.getRequiredColumns());
}
return allColumns;
}
@Override
default boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
for (Filter filter : getFilters()) {
if (!filter.supportsBitmapIndex(selector)) {
return false;
}
}
return true;
}
@Override
default boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
for (Filter f : getFilters()) {
if (!f.shouldUseBitmapIndex(selector)) {
return false;
}
}
return true;
}
@Override
default boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
{
for (Filter filter : getFilters()) {
if (!filter.supportsSelectivityEstimation(columnSelector, indexSelector)) {
return false;
}
}
return true;
}
}

116
processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java
View File

@ -20,13 +20,15 @@
package org.apache.druid.query.filter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import com.google.common.collect.BoundType;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Range;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import com.google.common.collect.TreeRangeSet;
import com.google.common.primitives.Doubles;
import com.google.common.primitives.Floats;
@ -41,32 +43,38 @@ import javax.annotation.Nullable;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
public class BoundDimFilter implements DimFilter
{
private final String dimension;
@Nullable
private final String upper;
@Nullable
private final String lower;
private final boolean lowerStrict;
private final boolean upperStrict;
@Nullable
private final ExtractionFn extractionFn;
private final StringComparator ordering;
private final Supplier<DruidLongPredicate> longPredicateSupplier;
private final Supplier<DruidFloatPredicate> floatPredicateSupplier;
private final Supplier<DruidDoublePredicate> doublePredicateSupplier;
@Nullable
private final FilterTuning filterTuning;
@JsonCreator
public BoundDimFilter(
@JsonProperty("dimension") String dimension,
@JsonProperty("lower") String lower,
@JsonProperty("upper") String upper,
@JsonProperty("lowerStrict") Boolean lowerStrict,
@JsonProperty("upperStrict") Boolean upperStrict,
@Deprecated @JsonProperty("alphaNumeric") Boolean alphaNumeric,
@JsonProperty("extractionFn") ExtractionFn extractionFn,
@JsonProperty("ordering") StringComparator ordering
@JsonProperty("lower") @Nullable String lower,
@JsonProperty("upper") @Nullable String upper,
@JsonProperty("lowerStrict") @Nullable Boolean lowerStrict,
@JsonProperty("upperStrict") @Nullable Boolean upperStrict,
@Deprecated @JsonProperty("alphaNumeric") @Nullable Boolean alphaNumeric,
@JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn,
@JsonProperty("ordering") @Nullable StringComparator ordering,
@JsonProperty("filterTuning") @Nullable FilterTuning filterTuning
)
{
this.dimension = Preconditions.checkNotNull(dimension, "dimension can not be null");
@ -98,6 +106,22 @@ public class BoundDimFilter implements DimFilter
this.longPredicateSupplier = makeLongPredicateSupplier();
this.floatPredicateSupplier = makeFloatPredicateSupplier();
this.doublePredicateSupplier = makeDoublePredicateSupplier();
this.filterTuning = filterTuning;
}
@VisibleForTesting
public BoundDimFilter(
String dimension,
@Nullable String lower,
@Nullable String upper,
@Nullable Boolean lowerStrict,
@Nullable Boolean upperStrict,
@Nullable Boolean alphaNumeric,
@Nullable ExtractionFn extractionFn,
@Nullable StringComparator ordering
)
{
this(dimension, lower, upper, lowerStrict, upperStrict, alphaNumeric, extractionFn, ordering, null);
}
@JsonProperty
@ -106,12 +130,14 @@ public class BoundDimFilter implements DimFilter
return dimension;
}
@Nullable
@JsonProperty
public String getUpper()
{
return upper;
}
@Nullable
@JsonProperty
public String getLower()
{
@ -140,6 +166,7 @@ public class BoundDimFilter implements DimFilter
return upper != null;
}
@Nullable
@JsonProperty
public ExtractionFn getExtractionFn()
{
@ -152,6 +179,14 @@ public class BoundDimFilter implements DimFilter
return ordering;
}
@Nullable
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonProperty
public FilterTuning getFilterTuning()
{
return filterTuning;
}
public Supplier<DruidLongPredicate> getLongPredicateSupplier()
{
return longPredicateSupplier;
@ -249,9 +284,9 @@ public class BoundDimFilter implements DimFilter
}
@Override
public HashSet<String> getRequiredColumns()
public Set<String> getRequiredColumns()
{
return Sets.newHashSet(dimension);
return ImmutableSet.of(dimension);
}
@Override
@ -263,49 +298,36 @@ public class BoundDimFilter implements DimFilter
if (o == null || getClass() != o.getClass()) {
return false;
}
BoundDimFilter that = (BoundDimFilter) o;
if (isLowerStrict() != that.isLowerStrict()) {
return false;
}
if (isUpperStrict() != that.isUpperStrict()) {
return false;
}
if (!getDimension().equals(that.getDimension())) {
return false;
}
if (getUpper() != null ? !getUpper().equals(that.getUpper()) : that.getUpper() != null) {
return false;
}
if (getLower() != null ? !getLower().equals(that.getLower()) : that.getLower() != null) {
return false;
}
if (getExtractionFn() != null
? !getExtractionFn().equals(that.getExtractionFn())
: that.getExtractionFn() != null) {
return false;
}
return getOrdering().equals(that.getOrdering());
return lowerStrict == that.lowerStrict &&
upperStrict == that.upperStrict &&
dimension.equals(that.dimension) &&
Objects.equals(upper, that.upper) &&
Objects.equals(lower, that.lower) &&
Objects.equals(extractionFn, that.extractionFn) &&
Objects.equals(ordering, that.ordering) &&
Objects.equals(filterTuning, that.filterTuning);
}
@Override
public int hashCode()
{
int result = getDimension().hashCode();
result = 31 * result + (getUpper() != null ? getUpper().hashCode() : 0);
result = 31 * result + (getLower() != null ? getLower().hashCode() : 0);
result = 31 * result + (isLowerStrict() ? 1 : 0);
result = 31 * result + (isUpperStrict() ? 1 : 0);
result = 31 * result + (getExtractionFn() != null ? getExtractionFn().hashCode() : 0);
result = 31 * result + getOrdering().hashCode();
return result;
return Objects.hash(
dimension,
upper,
lower,
lowerStrict,
upperStrict,
extractionFn,
ordering,
filterTuning
);
}
@Override
public String toString()
{
final StringBuilder builder = new StringBuilder();
final DimFilterToStringBuilder builder = new DimFilterToStringBuilder();
if (lower != null) {
builder.append(lower);
@ -316,11 +338,7 @@ public class BoundDimFilter implements DimFilter
}
}
if (extractionFn != null) {
builder.append(StringUtils.format("%s(%s)", extractionFn, dimension));
} else {
builder.append(dimension);
}
builder.appendDimension(dimension, extractionFn);
if (!ordering.equals(StringComparators.LEXICOGRAPHIC)) {
builder.append(StringUtils.format(" as %s", ordering.toString()));
@ -335,7 +353,7 @@ public class BoundDimFilter implements DimFilter
builder.append(upper);
}
return builder.toString();
return builder.appendFilterTuning(filterTuning).build();
}
private Supplier<DruidLongPredicate> makeLongPredicateSupplier()

25
processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java
View File

@ -24,13 +24,13 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.query.dimension.DimensionSpec;
import org.apache.druid.segment.filter.ColumnComparisonFilter;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
/**
@ -94,12 +94,16 @@ public class ColumnComparisonDimFilter implements DimFilter
if (o == null || getClass() != o.getClass()) {
return false;
}
ColumnComparisonDimFilter that = (ColumnComparisonDimFilter) o;
return dimensions.equals(that.dimensions);
}
@Override
public int hashCode()
{
return Objects.hash(dimensions);
}
@Override
public RangeSet<String> getDimensionRangeSet(String dimension)
{
@ -107,17 +111,8 @@ public class ColumnComparisonDimFilter implements DimFilter
}
@Override
public HashSet<String> getRequiredColumns()
public Set<String> getRequiredColumns()
{
return Sets.newHashSet(dimensions.stream()
.map(DimensionSpec::getDimension)
.collect(Collectors.toSet())
);
}
@Override
public int hashCode()
{
return 31 * dimensions.hashCode();
return dimensions.stream().map(DimensionSpec::getDimension).collect(Collectors.toSet());
}
}

71
processing/src/main/java/org/apache/druid/query/filter/DimFilter.java
View File

@ -23,8 +23,10 @@ import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.google.common.collect.RangeSet;
import org.apache.druid.java.util.common.Cacheable;
import org.apache.druid.query.extraction.ExtractionFn;
import java.util.HashSet;
import javax.annotation.Nullable;
import java.util.Set;
/**
*/
@ -81,5 +83,70 @@ public interface DimFilter extends Cacheable
/**
* @return a HashSet that represents all columns' name which the DimFilter required to do filter.
*/
HashSet<String> getRequiredColumns();
Set<String> getRequiredColumns();
/**
* Wrapper for {@link StringBuilder} to re-use common patterns in custom {@link DimFilter#toString()} implementations
*/
class DimFilterToStringBuilder
{
private final StringBuilder builder;
public DimFilterToStringBuilder()
{
this.builder = new StringBuilder();
}
/**
* Append dimension name OR {@link ExtractionFn#toString()} with dimension wrapped in parenthesis
*/
DimFilterToStringBuilder appendDimension(String dimension, @Nullable ExtractionFn extractionFn)
{
if (extractionFn != null) {
builder.append(extractionFn).append("(");
}
builder.append(dimension);
if (extractionFn != null) {
builder.append(")");
}
return this;
}
/**
* Add "=" expression
*/
DimFilterToStringBuilder appendEquals(String value)
{
builder.append(" = ").append(value);
return this;
}
/**
* Add filter tuning to {@link #builder} if tuning exists
*/
DimFilterToStringBuilder appendFilterTuning(@Nullable FilterTuning tuning)
{
if (tuning != null) {
builder.append(" (filterTuning=").append(tuning).append(")");
}
return this;
}
/**
* Generic passthrough to {@link StringBuilder#append}
*/
<T> DimFilterToStringBuilder append(T s)
{
builder.append(s);
return this;
}
public String build()
{
return builder.toString();
}
}
}

56
processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java
View File

@ -21,41 +21,61 @@ package org.apache.druid.query.filter;
import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprMacroTable;
import org.apache.druid.math.expr.Parser;
import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.segment.filter.ExpressionFilter;
import java.util.HashSet;
import javax.annotation.Nullable;
import java.util.Objects;
import java.util.Set;
public class ExpressionDimFilter implements DimFilter
{
private final String expression;
private final Supplier<Expr> parsed;
@Nullable
private final FilterTuning filterTuning;
@JsonCreator
public ExpressionDimFilter(
@JsonProperty("expression") final String expression,
@JsonProperty("filterTuning") @Nullable final FilterTuning filterTuning,
@JacksonInject ExprMacroTable macroTable
)
{
this.expression = expression;
this.filterTuning = filterTuning;
this.parsed = Suppliers.memoize(() -> Parser.parse(expression, macroTable));
}
@VisibleForTesting
public ExpressionDimFilter(final String expression, ExprMacroTable macroTable)
{
this(expression, null, macroTable);
}
@JsonProperty
public String getExpression()
{
return expression;
}
@Nullable
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonProperty
public FilterTuning getFilterTuning()
{
return filterTuning;
}
@Override
public DimFilter optimize()
{
@ -65,7 +85,7 @@ public class ExpressionDimFilter implements DimFilter
@Override
public Filter toFilter()
{
return new ExpressionFilter(parsed);
return new ExpressionFilter(parsed, filterTuning);
}
@Override
@ -75,9 +95,9 @@ public class ExpressionDimFilter implements DimFilter
}
@Override
public HashSet<String> getRequiredColumns()
public Set<String> getRequiredColumns()
{
return Sets.newHashSet(parsed.get().analyzeInputs().getRequiredBindings());
return parsed.get().analyzeInputs().getRequiredBindings();
}
@Override
@ -89,7 +109,16 @@ public class ExpressionDimFilter implements DimFilter
}
@Override
public boolean equals(final Object o)
public String toString()
{
return "ExpressionDimFilter{" +
"expression='" + expression + '\'' +
", filterTuning=" + filterTuning +
'}';
}
@Override
public boolean equals(Object o)
{
if (this == o) {
return true;
@ -97,21 +126,14 @@ public class ExpressionDimFilter implements DimFilter
if (o == null || getClass() != o.getClass()) {
return false;
}
final ExpressionDimFilter that = (ExpressionDimFilter) o;
return Objects.equals(expression, that.expression);
ExpressionDimFilter that = (ExpressionDimFilter) o;
return expression.equals(that.expression) &&
Objects.equals(filterTuning, that.filterTuning);
}
@Override
public int hashCode()
{
return Objects.hash(expression);
}
@Override
public String toString()
{
return "ExpressionDimFilter{" +
"expression='" + expression + '\'' +
'}';
return Objects.hash(expression, filterTuning);
}
}

8
processing/src/main/java/org/apache/druid/query/filter/ExtractionDimFilter.java
View File

@ -22,13 +22,13 @@ package org.apache.druid.query.filter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.extraction.ExtractionFn;
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.Set;
/**
* This class is deprecated, use SelectorDimFilter instead: {@link SelectorDimFilter}
@ -113,9 +113,9 @@ public class ExtractionDimFilter implements DimFilter
}
@Override
public HashSet<String> getRequiredColumns()
public Set<String> getRequiredColumns()
{
return Sets.newHashSet(dimension);
return ImmutableSet.of(dimension);
}
@Override

37
processing/src/main/java/org/apache/druid/query/filter/Filter.java
View File

@ -28,6 +28,8 @@ import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import java.util.Set;
public interface Filter
{
/**
@ -100,8 +102,12 @@ public interface Filter
}
/**
* Indicates whether this filter can return a bitmap index for filtering, based on
* the information provided by the input BitmapIndexSelector.
* Indicates whether this filter can return a bitmap index for filtering, based on the information provided by the
* input {@link BitmapIndexSelector}.
*
* Returning a value of true here guarantees that {@link #getBitmapIndex(BitmapIndexSelector)} will return a non-null
* {@link BitmapIndexSelector}, and also that all columns specified in {@link #getRequiredColumns()} have a bitmap
* index retrievable via {@link BitmapIndexSelector#getBitmapIndex(String)}.
*
* @param selector Object used to retrieve bitmap indexes
*
@ -109,6 +115,27 @@ public interface Filter
*/
boolean supportsBitmapIndex(BitmapIndexSelector selector);
/**
* Determine if a filter *should* use a bitmap index based on information collected from the supplied
* {@link BitmapIndexSelector}. This method differs from {@link #supportsBitmapIndex(BitmapIndexSelector)} in that
* the former only indicates if a bitmap index is available and {@link #getBitmapIndex(BitmapIndexSelector)} may be
* used.
*
* If shouldUseFilter(selector) returns true, {@link #supportsBitmapIndex} must also return true when called with the
* same selector object. Returning a value of true here guarantees that {@link #getBitmapIndex(BitmapIndexSelector)}
* will return a non-null {@link BitmapIndexSelector}, and also that all columns specified in
* {@link #getRequiredColumns()} have a bitmap index retrievable via
* {@link BitmapIndexSelector#getBitmapIndex(String)}.
*
* Implementations of this methods typically consider a {@link FilterTuning} to make decisions about when to
* use an available index. A "standard" implementation of this is available to all {@link Filter} implementations in
* {@link org.apache.druid.segment.filter.Filters#shouldUseBitmapIndex(Filter, BitmapIndexSelector, FilterTuning)}.
*
* @param selector Object used to retrieve bitmap indexes and provide information about the column
*
* @return true if this Filter should provide a bitmap index using the selector, false otherwise.
*/
boolean shouldUseBitmapIndex(BitmapIndexSelector selector);
/**
* Indicates whether this filter supports selectivity estimation.
@ -129,4 +156,10 @@ public interface Filter
{
return false;
}
/**
* Set of columns used by a filter. If {@link #supportsBitmapIndex} returns true, all columns returned by this method
* can be expected to have a bitmap index retrievable via {@link BitmapIndexSelector#getBitmapIndex(String)}
*/
Set<String> getRequiredColumns();
}

121
processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java Normal file
View File

@ -0,0 +1,121 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.query.filter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import javax.annotation.Nullable;
import java.util.Objects;
/**
* This class provides a mechansim to influence whether or not indexes are used for a {@link Filter} during processing
* by {@link org.apache.druid.segment.QueryableIndexStorageAdapter#analyzeFilter} (i.e. will a {@link Filter} be a "pre"
* filter in which we union indexes for all values that match the filter to create a
* {@link org.apache.druid.segment.BitmapOffset}/{@link org.apache.druid.segment.vector.BitmapVectorOffset}, or will it
* be used as a "post" filter and evaluated while scanning row values from the
* {@link org.apache.druid.segment.FilteredOffset}/{@link org.apache.druid.segment.vector.FilteredVectorOffset}.
*
* This is currently only manually supplied by the user by adding to a {@link DimFilter} which will pass through to the
* {@link Filter} implementation. The main purpose at this time is to facilitate experimentation so that someday we can
* have {@link Filter} implementations intelligently, automatically use sensible defaults based on things like
* cardinality and who yet knows what additional information.
*
* It can also be used for advanced users to manually control which filters will be "pre" and "post" filters as
* described above to allow skipping indexes in known cases where filters are expensive (mostly high cardinality columns
* with expensive filters).
*
* As such, it is currently undocumented in user facing documentation on purpose, but whatever this turns into once more
* automatic usage of this is in place, should be documented in a future release.
*/
public class FilterTuning
{
public static FilterTuning createDefault(Filter filter, BitmapIndexSelector selector)
{
return new FilterTuning(filter.supportsBitmapIndex(selector), null, null);
}
private final boolean useBitmapIndex;
private final int minCardinalityToUseBitmapIndex;
private final int maxCardinalityToUseBitmapIndex;
@JsonCreator
public FilterTuning(
@JsonProperty("useBitmapIndex") @Nullable Boolean useBitmapIndex,
@JsonProperty("minCardinalityToUseBitmapIndex") @Nullable Integer minCardinalityToUseBitmapIndex,
@JsonProperty("useIndexMaximumCardinalityThreshold") @Nullable Integer maxCardinalityToUseBitmapIndex
)
{
this.useBitmapIndex = useBitmapIndex != null ? useBitmapIndex : true;
this.minCardinalityToUseBitmapIndex =
minCardinalityToUseBitmapIndex != null ? minCardinalityToUseBitmapIndex : 0;
this.maxCardinalityToUseBitmapIndex =
maxCardinalityToUseBitmapIndex != null ? maxCardinalityToUseBitmapIndex : Integer.MAX_VALUE;
}
@JsonProperty
public boolean getUseBitmapIndex()
{
return useBitmapIndex;
}
@JsonProperty
public int getMinCardinalityToUseBitmapIndex()
{
return minCardinalityToUseBitmapIndex;
}
@JsonProperty
public int getMaxCardinalityToUseBitmapIndex()
{
return maxCardinalityToUseBitmapIndex;
}
@Override
public boolean equals(Object o)
{
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
FilterTuning that = (FilterTuning) o;
return Objects.equals(useBitmapIndex, that.useBitmapIndex) &&
Objects.equals(minCardinalityToUseBitmapIndex, that.minCardinalityToUseBitmapIndex) &&
Objects.equals(maxCardinalityToUseBitmapIndex, that.maxCardinalityToUseBitmapIndex);
}
@Override
public int hashCode()
{
return Objects.hash(useBitmapIndex, minCardinalityToUseBitmapIndex, maxCardinalityToUseBitmapIndex);
}
@Override
public String toString()
{
return "FilterTuning{" +
"useBitmapIndex=" + useBitmapIndex +
", minCardinalityToUseBitmapIndex=" + minCardinalityToUseBitmapIndex +
", maxCardinalityToUseBitmapIndex=" + maxCardinalityToUseBitmapIndex +
'}';
}
}

95
processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java
View File

@ -20,14 +20,16 @@
package org.apache.druid.query.filter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Range;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import com.google.common.collect.TreeRangeSet;
import com.google.common.primitives.Doubles;
import com.google.common.primitives.Floats;
@ -45,10 +47,10 @@ import org.apache.druid.query.lookup.LookupExtractor;
import org.apache.druid.segment.DimensionHandlerUtils;
import org.apache.druid.segment.filter.InFilter;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
@ -64,7 +66,10 @@ public class InDimFilter implements DimFilter
// Values can contain `null` object
private final SortedSet<String> values;
private final String dimension;
@Nullable
private final ExtractionFn extractionFn;
@Nullable
private final FilterTuning filterTuning;
private final Supplier<DruidLongPredicate> longPredicateSupplier;
private final Supplier<DruidFloatPredicate> floatPredicateSupplier;
private final Supplier<DruidDoublePredicate> doublePredicateSupplier;
@ -73,7 +78,8 @@ public class InDimFilter implements DimFilter
public InDimFilter(
@JsonProperty("dimension") String dimension,
@JsonProperty("values") Collection<String> values,
@JsonProperty("extractionFn") ExtractionFn extractionFn
@JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn,
@JsonProperty("filterTuning") @Nullable FilterTuning filterTuning
)
{
Preconditions.checkNotNull(dimension, "dimension can not be null");
@ -85,11 +91,18 @@ public class InDimFilter implements DimFilter
}
this.dimension = dimension;
this.extractionFn = extractionFn;
this.filterTuning = filterTuning;
this.longPredicateSupplier = getLongPredicateSupplier();
this.floatPredicateSupplier = getFloatPredicateSupplier();
this.doublePredicateSupplier = getDoublePredicateSupplier();
}
@VisibleForTesting
public InDimFilter(String dimension, Collection<String> values, @Nullable ExtractionFn extractionFn)
{
this(dimension, values, extractionFn, null);
}
@JsonProperty
public String getDimension()
{
@ -102,12 +115,21 @@ public class InDimFilter implements DimFilter
return values;
}
@Nullable
@JsonProperty
public ExtractionFn getExtractionFn()
{
return extractionFn;
}
@Nullable
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonProperty
public FilterTuning getFilterTuning()
{
return filterTuning;
}
@Override
public byte[] getCacheKey()
{
@ -133,7 +155,7 @@ public class InDimFilter implements DimFilter
{
InDimFilter inFilter = optimizeLookup();
if (inFilter.values.size() == 1) {
return new SelectorDimFilter(inFilter.dimension, inFilter.values.first(), inFilter.getExtractionFn());
return new SelectorDimFilter(inFilter.dimension, inFilter.values.first(), inFilter.getExtractionFn(), filterTuning);
}
return inFilter;
}
@ -169,7 +191,7 @@ public class InDimFilter implements DimFilter
if (keys.isEmpty()) {
return this;
} else {
return new InDimFilter(dimension, keys, null);
return new InDimFilter(dimension, keys, null, filterTuning);
}
}
return this;
@ -184,7 +206,8 @@ public class InDimFilter implements DimFilter
longPredicateSupplier,
floatPredicateSupplier,
doublePredicateSupplier,
extractionFn
extractionFn,
filterTuning
);
}
@ -210,9 +233,21 @@ public class InDimFilter implements DimFilter
}
@Override
public HashSet<String> getRequiredColumns()
public Set<String> getRequiredColumns()
{
return Sets.newHashSet(dimension);
return ImmutableSet.of(dimension);
}
@Override
public String toString()
{
final DimFilterToStringBuilder builder = new DimFilterToStringBuilder();
return builder.appendDimension(dimension, extractionFn)
.append(" IN (")
.append(Joiner.on(", ").join(Iterables.transform(values, StringUtils::nullToEmptyNonDruidDataString)))
.append(")")
.appendFilterTuning(filterTuning)
.build();
}
@Override
@ -224,51 +259,17 @@ public class InDimFilter implements DimFilter
if (o == null || getClass() != o.getClass()) {
return false;
}
InDimFilter that = (InDimFilter) o;
if (values != null ? !values.equals(that.values) : that.values != null) {
return false;
}
if (!dimension.equals(that.dimension)) {
return false;
}
return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null;
return values.equals(that.values) &&
dimension.equals(that.dimension) &&
Objects.equals(extractionFn, that.extractionFn) &&
Objects.equals(filterTuning, that.filterTuning);
}
@Override
public int hashCode()
{
int result = values != null ? values.hashCode() : 0;
result = 31 * result + dimension.hashCode();
result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0);
return result;
}
@Override
public String toString()
{
final StringBuilder builder = new StringBuilder();
if (extractionFn != null) {
builder.append(extractionFn).append("(");
}
builder.append(dimension);
if (extractionFn != null) {
builder.append(")");
}
builder.append(" IN (")
.append(
Joiner.on(", ").join(
Iterables.transform(values, input -> StringUtils.nullToEmptyNonDruidDataString(input))
)
)
.append(")");
return builder.toString();
return Objects.hash(values, dimension, extractionFn, filterTuning);
}
// As the set of filtered values can be large, parsing them as longs should be done only if needed, and only once.

60
processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java
View File

@ -20,10 +20,12 @@
package org.apache.druid.query.filter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import com.google.common.primitives.Longs;
import org.apache.druid.java.util.common.JodaUtils;
import org.apache.druid.java.util.common.Pair;
@ -32,25 +34,31 @@ import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.query.ordering.StringComparators;
import org.joda.time.Interval;
import javax.annotation.Nullable;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
public class IntervalDimFilter implements DimFilter
{
private final List<Interval> intervals;
private final List<Pair<Long, Long>> intervalLongs;
private final String dimension;
@Nullable
private final ExtractionFn extractionFn;
@Nullable
private final FilterTuning filterTuning;
private final OrDimFilter convertedFilter;
@JsonCreator
public IntervalDimFilter(
@JsonProperty("dimension") String dimension,
@JsonProperty("intervals") List<Interval> intervals,
@JsonProperty("extractionFn") ExtractionFn extractionFn
@JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn,
@JsonProperty("filterTuning") @Nullable FilterTuning filterTuning
)
{
Preconditions.checkNotNull(dimension, "dimension can not be null");
@ -59,10 +67,17 @@ public class IntervalDimFilter implements DimFilter
this.dimension = dimension;
this.intervals = Collections.unmodifiableList(JodaUtils.condenseIntervals(intervals));
this.extractionFn = extractionFn;
this.filterTuning = filterTuning;
this.intervalLongs = makeIntervalLongs();
this.convertedFilter = new OrDimFilter(makeBoundDimFilters());
}
@VisibleForTesting
public IntervalDimFilter(String dimension, List<Interval> intervals, @Nullable ExtractionFn extractionFn)
{
this(dimension, intervals, extractionFn, null);
}
@JsonProperty
public String getDimension()
{
@ -75,12 +90,21 @@ public class IntervalDimFilter implements DimFilter
return intervals;
}
@Nullable
@JsonProperty
public ExtractionFn getExtractionFn()
{
return extractionFn;
}
@Nullable
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonProperty
public FilterTuning getFilterTuning()
{
return filterTuning;
}
@Override
public byte[] getCacheKey()
{
@ -125,9 +149,9 @@ public class IntervalDimFilter implements DimFilter
}
@Override
public HashSet<String> getRequiredColumns()
public Set<String> getRequiredColumns()
{
return Sets.newHashSet(dimension);
return ImmutableSet.of(dimension);
}
@Override
@ -139,28 +163,17 @@ public class IntervalDimFilter implements DimFilter
if (o == null || getClass() != o.getClass()) {
return false;
}
IntervalDimFilter that = (IntervalDimFilter) o;
if (!getIntervals().equals(that.getIntervals())) {
return false;
}
if (!getDimension().equals(that.getDimension())) {
return false;
}
return getExtractionFn() != null
? getExtractionFn().equals(that.getExtractionFn())
: that.getExtractionFn() == null;
return intervals.equals(that.intervals) &&
dimension.equals(that.dimension) &&
Objects.equals(extractionFn, that.extractionFn) &&
Objects.equals(filterTuning, that.filterTuning);
}
@Override
public int hashCode()
{
int result = getIntervals().hashCode();
result = 31 * result + getDimension().hashCode();
result = 31 * result + (getExtractionFn() != null ? getExtractionFn().hashCode() : 0);
return result;
return Objects.hash(intervals, dimension, extractionFn, filterTuning);
}
@Override
@ -173,7 +186,7 @@ public class IntervalDimFilter implements DimFilter
{
List<Pair<Long, Long>> intervalLongs = new ArrayList<>();
for (Interval interval : intervals) {
intervalLongs.add(new Pair<Long, Long>(interval.getStartMillis(), interval.getEndMillis()));
intervalLongs.add(new Pair<>(interval.getStartMillis(), interval.getEndMillis()));
}
return intervalLongs;
}
@ -190,7 +203,8 @@ public class IntervalDimFilter implements DimFilter
true,
null,
extractionFn,
StringComparators.NUMERIC
StringComparators.NUMERIC,
filterTuning
);
boundDimFilters.add(boundDimFilter);
}

83
processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java
View File

@ -21,11 +21,13 @@ package org.apache.druid.query.filter;
import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.js.JavaScriptConfig;
import org.apache.druid.query.extraction.ExtractionFn;
@ -36,14 +38,19 @@ import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import javax.annotation.Nullable;
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
public class JavaScriptDimFilter implements DimFilter
{
private final String dimension;
private final String function;
@Nullable
private final ExtractionFn extractionFn;
@Nullable
private final FilterTuning filterTuning;
private final JavaScriptConfig config;
/**
@ -61,7 +68,8 @@ public class JavaScriptDimFilter implements DimFilter
public JavaScriptDimFilter(
@JsonProperty("dimension") String dimension,
@JsonProperty("function") String function,
@JsonProperty("extractionFn") ExtractionFn extractionFn,
@JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn,
@JsonProperty("filterTuning") @Nullable FilterTuning filterTuning,
@JacksonInject JavaScriptConfig config
)
{
@ -70,9 +78,21 @@ public class JavaScriptDimFilter implements DimFilter
this.dimension = dimension;
this.function = function;
this.extractionFn = extractionFn;
this.filterTuning = filterTuning;
this.config = config;
}
@VisibleForTesting
public JavaScriptDimFilter(
String dimension,
String function,
@Nullable ExtractionFn extractionFn,
JavaScriptConfig config
)
{
this(dimension, function, extractionFn, null, config);
}
@JsonProperty
public String getDimension()
{
@ -85,12 +105,21 @@ public class JavaScriptDimFilter implements DimFilter
return function;
}
@Nullable
@JsonProperty
public ExtractionFn getExtractionFn()
{
return extractionFn;
}
@Nullable
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonProperty
public FilterTuning getFilterTuning()
{
return filterTuning;
}
@Override
public byte[] getCacheKey()
{
@ -118,7 +147,7 @@ public class JavaScriptDimFilter implements DimFilter
public Filter toFilter()
{
JavaScriptPredicateFactory predicateFactory = getPredicateFactory();
return new JavaScriptFilter(dimension, predicateFactory);
return new JavaScriptFilter(dimension, predicateFactory, filterTuning);
}
/**
@ -152,19 +181,9 @@ public class JavaScriptDimFilter implements DimFilter
}
@Override
public HashSet<String> getRequiredColumns()
public Set<String> getRequiredColumns()
{
return Sets.newHashSet(dimension);
}
@Override
public String toString()
{
return "JavaScriptDimFilter{" +
"dimension='" + dimension + '\'' +
", function='" + function + '\'' +
", extractionFn='" + extractionFn + '\'' +
'}';
return ImmutableSet.of(dimension);
}
@Override
@ -173,29 +192,31 @@ public class JavaScriptDimFilter implements DimFilter
if (this == o) {
return true;
}
if (!(o instanceof JavaScriptDimFilter)) {
if (o == null || getClass() != o.getClass()) {
return false;
}
JavaScriptDimFilter that = (JavaScriptDimFilter) o;
if (!dimension.equals(that.dimension)) {
return false;
}
if (!function.equals(that.function)) {
return false;
}
return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null;
return dimension.equals(that.dimension) &&
function.equals(that.function) &&
Objects.equals(extractionFn, that.extractionFn) &&
Objects.equals(filterTuning, that.filterTuning);
}
@Override
public int hashCode()
{
int result = dimension.hashCode();
result = 31 * result + function.hashCode();
result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0);
return result;
return Objects.hash(dimension, function, extractionFn, filterTuning);
}
@Override
public String toString()
{
return "JavaScriptDimFilter{" +
"dimension='" + dimension + '\'' +
", function='" + function + '\'' +
", extractionFn=" + extractionFn +
", filterTuning=" + filterTuning +
'}';
}
public static class JavaScriptPredicateFactory implements DruidPredicateFactory

267
processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java
View File

@ -20,11 +20,13 @@
package org.apache.druid.query.filter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import com.google.common.io.BaseEncoding;
import com.google.common.primitives.Chars;
import org.apache.druid.common.config.NullHandling;
@ -35,7 +37,8 @@ import org.apache.druid.segment.filter.LikeFilter;
import javax.annotation.Nullable;
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Pattern;
public class LikeDimFilter implements DimFilter
@ -47,21 +50,27 @@ public class LikeDimFilter implements DimFilter
private final String dimension;
private final String pattern;
@Nullable
private final Character escapeChar;
@Nullable
private final ExtractionFn extractionFn;
@Nullable
private final FilterTuning filterTuning;
private final LikeMatcher likeMatcher;
@JsonCreator
public LikeDimFilter(
@JsonProperty("dimension") final String dimension,
@JsonProperty("pattern") final String pattern,
@JsonProperty("escape") final String escape,
@JsonProperty("extractionFn") final ExtractionFn extractionFn
@JsonProperty("escape") @Nullable final String escape,
@JsonProperty("extractionFn") @Nullable final ExtractionFn extractionFn,
@JsonProperty("filterTuning") @Nullable final FilterTuning filterTuning
)
{
this.dimension = Preconditions.checkNotNull(dimension, "dimension");
this.pattern = Preconditions.checkNotNull(pattern, "pattern");
this.extractionFn = extractionFn;
this.filterTuning = filterTuning;
if (escape != null && escape.length() != 1) {
throw new IllegalArgumentException("Escape must be null or a single character");
@ -72,6 +81,129 @@ public class LikeDimFilter implements DimFilter
this.likeMatcher = LikeMatcher.from(pattern, this.escapeChar);
}
@VisibleForTesting
public LikeDimFilter(
final String dimension,
final String pattern,
@Nullable final String escape,
@Nullable final ExtractionFn extractionFn
)
{
this(dimension, pattern, escape, extractionFn, null);
}
@JsonProperty
public String getDimension()
{
return dimension;
}
@JsonProperty
public String getPattern()
{
return pattern;
}
@Nullable
@JsonProperty
public String getEscape()
{
return escapeChar != null ? escapeChar.toString() : null;
}
@Nullable
@JsonProperty
public ExtractionFn getExtractionFn()
{
return extractionFn;
}
@Nullable
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonProperty
public FilterTuning getFilterTuning()
{
return filterTuning;
}
@Override
public byte[] getCacheKey()
{
final byte[] dimensionBytes = StringUtils.toUtf8(dimension);
final byte[] patternBytes = StringUtils.toUtf8(pattern);
final byte[] escapeBytes = escapeChar == null ? new byte[0] : Chars.toByteArray(escapeChar);
final byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey();
final int sz = 4 + dimensionBytes.length + patternBytes.length + escapeBytes.length + extractionFnBytes.length;
return ByteBuffer.allocate(sz)
.put(DimFilterUtils.LIKE_CACHE_ID)
.put(dimensionBytes)
.put(DimFilterUtils.STRING_SEPARATOR)
.put(patternBytes)
.put(DimFilterUtils.STRING_SEPARATOR)
.put(escapeBytes)
.put(DimFilterUtils.STRING_SEPARATOR)
.put(extractionFnBytes)
.array();
}
@Override
public DimFilter optimize()
{
return this;
}
@Override
public Filter toFilter()
{
return new LikeFilter(dimension, extractionFn, likeMatcher, filterTuning);
}
@Override
public RangeSet<String> getDimensionRangeSet(String dimension)
{
return null;
}
@Override
public Set<String> getRequiredColumns()
{
return ImmutableSet.of(dimension);
}
@Override
public boolean equals(Object o)
{
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
LikeDimFilter that = (LikeDimFilter) o;
return dimension.equals(that.dimension) &&
pattern.equals(that.pattern) &&
Objects.equals(escapeChar, that.escapeChar) &&
Objects.equals(extractionFn, that.extractionFn) &&
Objects.equals(filterTuning, that.filterTuning);
}
@Override
public int hashCode()
{
return Objects.hash(dimension, pattern, escapeChar, extractionFn, filterTuning);
}
@Override
public String toString()
{
final DimFilterToStringBuilder builder = new DimFilterToStringBuilder();
builder.appendDimension(dimension, extractionFn).append(" LIKE '").append(pattern).append("'");
if (escapeChar != null) {
builder.append(" ESCAPE '").append(escapeChar).append("'");
}
return builder.appendFilterTuning(filterTuning).build();
}
public static class LikeMatcher
{
public enum SuffixMatch
@ -232,131 +364,4 @@ public class LikeDimFilter implements DimFilter
return suffixMatch;
}
}
@JsonProperty
public String getDimension()
{
return dimension;
}
@JsonProperty
public String getPattern()
{
return pattern;
}
@JsonProperty
public String getEscape()
{
return escapeChar != null ? escapeChar.toString() : null;
}
@JsonProperty
public ExtractionFn getExtractionFn()
{
return extractionFn;
}
@Override
public byte[] getCacheKey()
{
final byte[] dimensionBytes = StringUtils.toUtf8(dimension);
final byte[] patternBytes = StringUtils.toUtf8(pattern);
final byte[] escapeBytes = escapeChar == null ? new byte[0] : Chars.toByteArray(escapeChar);
final byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey();
final int sz = 4 + dimensionBytes.length + patternBytes.length + escapeBytes.length + extractionFnBytes.length;
return ByteBuffer.allocate(sz)
.put(DimFilterUtils.LIKE_CACHE_ID)
.put(dimensionBytes)
.put(DimFilterUtils.STRING_SEPARATOR)
.put(patternBytes)
.put(DimFilterUtils.STRING_SEPARATOR)
.put(escapeBytes)
.put(DimFilterUtils.STRING_SEPARATOR)
.put(extractionFnBytes)
.array();
}
@Override
public DimFilter optimize()
{
return this;
}
@Override
public Filter toFilter()
{
return new LikeFilter(dimension, extractionFn, likeMatcher);
}
@Override
public RangeSet<String> getDimensionRangeSet(String dimension)
{
return null;
}
@Override
public HashSet<String> getRequiredColumns()
{
return Sets.newHashSet(dimension);
}
@Override
public boolean equals(Object o)
{
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
LikeDimFilter that = (LikeDimFilter) o;
if (dimension != null ? !dimension.equals(that.dimension) : that.dimension != null) {
return false;
}
if (pattern != null ? !pattern.equals(that.pattern) : that.pattern != null) {
return false;
}
if (escapeChar != null ? !escapeChar.equals(that.escapeChar) : that.escapeChar != null) {
return false;
}
return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null;
}
@Override
public int hashCode()
{
int result = dimension != null ? dimension.hashCode() : 0;
result = 31 * result + (pattern != null ? pattern.hashCode() : 0);
result = 31 * result + (escapeChar != null ? escapeChar.hashCode() : 0);
result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0);
return result;
}
@Override
public String toString()
{
final StringBuilder builder = new StringBuilder();
if (extractionFn != null) {
builder.append(extractionFn).append("(");
}
builder.append(dimension);
if (extractionFn != null) {
builder.append(")");
}
builder.append(" LIKE '").append(pattern).append("'");
if (escapeChar != null) {
builder.append(" ESCAPE '").append(escapeChar).append("'");
}
return builder.toString();
}
}

4
processing/src/main/java/org/apache/druid/query/filter/NotDimFilter.java
View File

@ -27,8 +27,8 @@ import com.google.common.collect.RangeSet;
import org.apache.druid.segment.filter.NotFilter;
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/**
*/
@ -102,7 +102,7 @@ public class NotDimFilter implements DimFilter
}
@Override
public HashSet<String> getRequiredColumns()
public Set<String> getRequiredColumns()
{
return field.getRequiredColumns();
}

3
processing/src/main/java/org/apache/druid/query/filter/OrDimFilter.java
View File

@ -33,6 +33,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/**
*/
@ -107,7 +108,7 @@ public class OrDimFilter implements DimFilter
}
@Override
public HashSet<String> getRequiredColumns()
public Set<String> getRequiredColumns()
{
HashSet<String> requiredColumns = new HashSet<>();
fields.forEach(field -> requiredColumns.addAll(field.getRequiredColumns()));

58
processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java
View File

@ -20,16 +20,20 @@
package org.apache.druid.query.filter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.segment.filter.RegexFilter;
import javax.annotation.Nullable;
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Pattern;
/**
@ -38,7 +42,10 @@ public class RegexDimFilter implements DimFilter
{
private final String dimension;
private final String pattern;
@Nullable
private final ExtractionFn extractionFn;
@Nullable
private final FilterTuning filterTuning;
private final Pattern compiledPattern;
@ -46,7 +53,8 @@ public class RegexDimFilter implements DimFilter
public RegexDimFilter(
@JsonProperty("dimension") String dimension,
@JsonProperty("pattern") String pattern,
@JsonProperty("extractionFn") ExtractionFn extractionFn
@JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn,
@JsonProperty("filterTuning") @Nullable FilterTuning filterTuning
)
{
Preconditions.checkArgument(dimension != null, "dimension must not be null");
@ -55,6 +63,13 @@ public class RegexDimFilter implements DimFilter
this.pattern = pattern;
this.extractionFn = extractionFn;
this.compiledPattern = Pattern.compile(pattern);
this.filterTuning = filterTuning;
}
@VisibleForTesting
public RegexDimFilter(String dimension, String pattern, @Nullable ExtractionFn extractionFn)
{
this(dimension, pattern, extractionFn, null);
}
@JsonProperty
@ -69,12 +84,21 @@ public class RegexDimFilter implements DimFilter
return pattern;
}
@Nullable
@JsonProperty
public ExtractionFn getExtractionFn()
{
return extractionFn;
}
@Nullable
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonProperty
public FilterTuning getFilterTuning()
{
return filterTuning;
}
@Override
public byte[] getCacheKey()
{
@ -101,7 +125,7 @@ public class RegexDimFilter implements DimFilter
@Override
public Filter toFilter()
{
return new RegexFilter(dimension, compiledPattern, extractionFn);
return new RegexFilter(dimension, compiledPattern, extractionFn, filterTuning);
}
@Override
@ -111,9 +135,9 @@ public class RegexDimFilter implements DimFilter
}
@Override
public HashSet<String> getRequiredColumns()
public Set<String> getRequiredColumns()
{
return Sets.newHashSet(dimension);
return ImmutableSet.of(dimension);
}
@Override
@ -123,6 +147,7 @@ public class RegexDimFilter implements DimFilter
"dimension='" + dimension + '\'' +
", pattern='" + pattern + '\'' +
", extractionFn='" + extractionFn + '\'' +
", filterTuning=" + filterTuning +
'}';
}
@ -132,28 +157,19 @@ public class RegexDimFilter implements DimFilter
if (this == o) {
return true;
}
if (!(o instanceof RegexDimFilter)) {
if (o == null || getClass() != o.getClass()) {
return false;
}
RegexDimFilter that = (RegexDimFilter) o;
if (!dimension.equals(that.dimension)) {
return false;
}
if (!pattern.equals(that.pattern)) {
return false;
}
return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null;
return dimension.equals(that.dimension) &&
pattern.equals(that.pattern) &&
Objects.equals(extractionFn, that.extractionFn) &&
Objects.equals(filterTuning, that.filterTuning);
}
@Override
public int hashCode()
{
int result = dimension.hashCode();
result = 31 * result + pattern.hashCode();
result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0);
return result;
return Objects.hash(dimension, pattern, extractionFn, filterTuning);
}
}

62
processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java
View File

@ -19,17 +19,21 @@
package org.apache.druid.query.filter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.query.search.SearchQuerySpec;
import org.apache.druid.segment.filter.SearchQueryFilter;
import javax.annotation.Nullable;
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
/**
*/
@ -37,12 +41,16 @@ public class SearchQueryDimFilter implements DimFilter
{
private final String dimension;
private final SearchQuerySpec query;
@Nullable
private final ExtractionFn extractionFn;
@Nullable
private final FilterTuning filterTuning;
public SearchQueryDimFilter(
@JsonProperty("dimension") String dimension,
@JsonProperty("query") SearchQuerySpec query,
@JsonProperty("extractionFn") ExtractionFn extractionFn
@JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn,
@JsonProperty("filterTuning") @Nullable FilterTuning filterTuning
)
{
Preconditions.checkArgument(dimension != null, "dimension must not be null");
@ -51,6 +59,17 @@ public class SearchQueryDimFilter implements DimFilter
this.dimension = dimension;
this.query = query;
this.extractionFn = extractionFn;
this.filterTuning = filterTuning;
}
@VisibleForTesting
public SearchQueryDimFilter(
String dimension,
SearchQuerySpec query,
@Nullable ExtractionFn extractionFn
)
{
this(dimension, query, extractionFn, null);
}
@JsonProperty
@ -65,12 +84,21 @@ public class SearchQueryDimFilter implements DimFilter
return query;
}
@Nullable
@JsonProperty
public ExtractionFn getExtractionFn()
{
return extractionFn;
}
@Nullable
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonProperty
public FilterTuning getFilterTuning()
{
return filterTuning;
}
@Override
public byte[] getCacheKey()
{
@ -97,7 +125,7 @@ public class SearchQueryDimFilter implements DimFilter
@Override
public Filter toFilter()
{
return new SearchQueryFilter(dimension, query, extractionFn);
return new SearchQueryFilter(dimension, query, extractionFn, filterTuning);
}
@Override
@ -107,9 +135,9 @@ public class SearchQueryDimFilter implements DimFilter
}
@Override
public HashSet<String> getRequiredColumns()
public Set<String> getRequiredColumns()
{
return Sets.newHashSet(dimension);
return ImmutableSet.of(dimension);
}
@Override
@ -119,6 +147,7 @@ public class SearchQueryDimFilter implements DimFilter
"dimension='" + dimension + '\'' +
", query=" + query +
", extractionFn='" + extractionFn + '\'' +
", filterTuning=" + filterTuning +
'}';
}
@ -128,28 +157,19 @@ public class SearchQueryDimFilter implements DimFilter
if (this == o) {
return true;
}
if (!(o instanceof SearchQueryDimFilter)) {
if (o == null || getClass() != o.getClass()) {
return false;
}
SearchQueryDimFilter that = (SearchQueryDimFilter) o;
if (!dimension.equals(that.dimension)) {
return false;
}
if (!query.equals(that.query)) {
return false;
}
return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null;
return dimension.equals(that.dimension) &&
query.equals(that.query) &&
Objects.equals(extractionFn, that.extractionFn) &&
Objects.equals(filterTuning, that.filterTuning);
}
@Override
public int hashCode()
{
int result = dimension.hashCode();
result = 31 * result + query.hashCode();
result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0);
return result;
return Objects.hash(dimension, query, extractionFn, filterTuning);
}
}

73
processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java
View File

@ -20,19 +20,19 @@
package org.apache.druid.query.filter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Range;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import com.google.common.collect.TreeRangeSet;
import com.google.common.primitives.Doubles;
import com.google.common.primitives.Floats;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.common.guava.GuavaUtils;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.segment.filter.DimensionPredicateFilter;
@ -40,8 +40,8 @@ import org.apache.druid.segment.filter.SelectorFilter;
import javax.annotation.Nullable;
import java.util.Collections;
import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
/**
*/
@ -51,7 +51,10 @@ public class SelectorDimFilter implements DimFilter
@Nullable
private final String value;
@Nullable
private final ExtractionFn extractionFn;
@Nullable
private final FilterTuning filterTuning;
private final Object initLock = new Object();
@ -63,7 +66,8 @@ public class SelectorDimFilter implements DimFilter
public SelectorDimFilter(
@JsonProperty("dimension") String dimension,
@JsonProperty("value") String value,
@JsonProperty("extractionFn") ExtractionFn extractionFn
@JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn,
@JsonProperty("filterTuning") @Nullable FilterTuning filterTuning
)
{
Preconditions.checkArgument(dimension != null, "dimension must not be null");
@ -71,6 +75,12 @@ public class SelectorDimFilter implements DimFilter
this.dimension = dimension;
this.value = NullHandling.emptyToNullIfNeeded(value);
this.extractionFn = extractionFn;
this.filterTuning = filterTuning;
}
public SelectorDimFilter(String dimension, String value, @Nullable ExtractionFn extractionFn)
{
this(dimension, value, extractionFn, null);
}
@Override
@ -90,14 +100,14 @@ public class SelectorDimFilter implements DimFilter
@Override
public DimFilter optimize()
{
return new InDimFilter(dimension, Collections.singletonList(value), extractionFn).optimize();
return new InDimFilter(dimension, Collections.singletonList(value), extractionFn, filterTuning).optimize();
}
@Override
public Filter toFilter()
{
if (extractionFn == null) {
return new SelectorFilter(dimension, value);
return new SelectorFilter(dimension, value, filterTuning);
} else {
final DruidPredicateFactory predicateFactory = new DruidPredicateFactory()
@ -129,7 +139,7 @@ public class SelectorDimFilter implements DimFilter
return druidDoublePredicate;
}
};
return new DimensionPredicateFilter(dimension, predicateFactory, extractionFn);
return new DimensionPredicateFilter(dimension, predicateFactory, extractionFn, filterTuning);
}
}
@ -139,26 +149,35 @@ public class SelectorDimFilter implements DimFilter
return dimension;
}
@Nullable
@JsonProperty
public String getValue()
{
return value;
}
@Nullable
@JsonProperty
public ExtractionFn getExtractionFn()
{
return extractionFn;
}
@Nullable
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonProperty
public FilterTuning getFilterTuning()
{
return filterTuning;
}
@Override
public String toString()
{
if (extractionFn != null) {
return StringUtils.format("%s(%s) = %s", extractionFn, dimension, value);
} else {
return StringUtils.format("%s = %s", dimension, value);
}
return new DimFilterToStringBuilder().appendDimension(dimension, extractionFn)
.appendEquals(value)
.appendFilterTuning(filterTuning)
.build();
}
@Override
@ -170,16 +189,17 @@ public class SelectorDimFilter implements DimFilter
if (o == null || getClass() != o.getClass()) {
return false;
}
SelectorDimFilter that = (SelectorDimFilter) o;
return dimension.equals(that.dimension) &&
Objects.equals(value, that.value) &&
Objects.equals(extractionFn, that.extractionFn) &&
Objects.equals(filterTuning, that.filterTuning);
}
if (!dimension.equals(that.dimension)) {
return false;
}
if (value != null ? !value.equals(that.value) : that.value != null) {
return false;
}
return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null;
@Override
public int hashCode()
{
return Objects.hash(dimension, value, extractionFn, filterTuning);
}
@Override
@ -201,18 +221,9 @@ public class SelectorDimFilter implements DimFilter
}
@Override
public HashSet<String> getRequiredColumns()
public Set<String> getRequiredColumns()
{
return Sets.newHashSet(dimension);
}
@Override
public int hashCode()
{
int result = dimension.hashCode();
result = 31 * result + (value != null ? value.hashCode() : 0);
result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0);
return result;
return ImmutableSet.of(dimension);
}

70
processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java
View File

@ -20,16 +20,20 @@
package org.apache.druid.query.filter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import org.apache.druid.collections.spatial.search.Bound;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.segment.filter.SpatialFilter;
import javax.annotation.Nullable;
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
/**
*/
@ -37,11 +41,14 @@ public class SpatialDimFilter implements DimFilter
{
private final String dimension;
private final Bound bound;
@Nullable
private final FilterTuning filterTuning;
@JsonCreator
public SpatialDimFilter(
@JsonProperty("dimension") String dimension,
@JsonProperty("bound") Bound bound
@JsonProperty("bound") Bound bound,
@JsonProperty("filterTuning") @Nullable FilterTuning filterTuning
)
{
Preconditions.checkArgument(dimension != null, "dimension must not be null");
@ -49,6 +56,13 @@ public class SpatialDimFilter implements DimFilter
this.dimension = dimension;
this.bound = bound;
this.filterTuning = filterTuning;
}
@VisibleForTesting
public SpatialDimFilter(String dimension, Bound bound)
{
this(dimension, bound, null);
}
@Override
@ -83,10 +97,18 @@ public class SpatialDimFilter implements DimFilter
return bound;
}
@Nullable
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonProperty
public FilterTuning getFilterTuning()
{
return filterTuning;
}
@Override
public Filter toFilter()
{
return new SpatialFilter(dimension, bound);
return new SpatialFilter(dimension, bound, filterTuning);
}
@Override
@ -96,9 +118,19 @@ public class SpatialDimFilter implements DimFilter
}
@Override
public HashSet<String> getRequiredColumns()
public Set<String> getRequiredColumns()
{
return Sets.newHashSet(dimension);
return ImmutableSet.of(dimension);
}
@Override
public String toString()
{
return "SpatialDimFilter{" +
"dimension='" + dimension + '\'' +
", bound=" + bound +
", filterTuning=" + filterTuning +
'}';
}
@Override
@ -110,33 +142,15 @@ public class SpatialDimFilter implements DimFilter
if (o == null || getClass() != o.getClass()) {
return false;
}
SpatialDimFilter that = (SpatialDimFilter) o;
if (bound != null ? !bound.equals(that.bound) : that.bound != null) {
return false;
}
if (dimension != null ? !dimension.equals(that.dimension) : that.dimension != null) {
return false;
}
return true;
return dimension.equals(that.dimension) &&
bound.equals(that.bound) &&
Objects.equals(filterTuning, that.filterTuning);
}
@Override
public int hashCode()
{
int result = dimension != null ? dimension.hashCode() : 0;
result = 31 * result + (bound != null ? bound.hashCode() : 0);
return result;
}
@Override
public String toString()
{
return "SpatialDimFilter{" +
"dimension='" + dimension + '\'' +
", bound=" + bound +
'}';
return Objects.hash(dimension, bound, filterTuning);
}
}

9
processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java
View File

@ -23,14 +23,13 @@ import com.google.common.collect.RangeSet;
import org.apache.druid.segment.filter.TrueFilter;
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.Collections;
import java.util.Set;
/**
*/
public class TrueDimFilter implements DimFilter
{
private static HashSet<String> REQUIRED_COLUMNS = new HashSet<>();
@Override
public byte[] getCacheKey()
{
@ -56,8 +55,8 @@ public class TrueDimFilter implements DimFilter
}
@Override
public HashSet<String> getRequiredColumns()
public Set<String> getRequiredColumns()
{
return REQUIRED_COLUMNS;
return Collections.emptySet();
}
}

6
processing/src/main/java/org/apache/druid/query/lookup/LookupExtractionFn.java
View File

@ -40,9 +40,9 @@ public class LookupExtractionFn extends FunctionalExtraction
public LookupExtractionFn(
@JsonProperty("lookup") final LookupExtractor lookup,
@JsonProperty("retainMissingValue") final boolean retainMissingValue,
@Nullable @JsonProperty("replaceMissingValueWith") final String replaceMissingValueWith,
@Nullable @JsonProperty("injective") final Boolean injective,
@Nullable @JsonProperty("optimize") final Boolean optimize
@JsonProperty("replaceMissingValueWith") @Nullable final String replaceMissingValueWith,
@JsonProperty("injective") @Nullable final Boolean injective,
@JsonProperty("optimize") @Nullable final Boolean optimize
)
{
super(

4
processing/src/main/java/org/apache/druid/query/lookup/RegisteredLookupExtractionFn.java
View File

@ -47,8 +47,8 @@ public class RegisteredLookupExtractionFn implements ExtractionFn
@JacksonInject LookupExtractorFactoryContainerProvider manager,
@JsonProperty("lookup") String lookup,
@JsonProperty("retainMissingValue") final boolean retainMissingValue,
@Nullable @JsonProperty("replaceMissingValueWith") final String replaceMissingValueWith,
@Nullable @JsonProperty("injective") final Boolean injective,
@JsonProperty("replaceMissingValueWith") @Nullable final String replaceMissingValueWith,
@JsonProperty("injective") @Nullable final Boolean injective,
@JsonProperty("optimize") Boolean optimize
)
{

2
processing/src/main/java/org/apache/druid/query/scan/ScanResultValue.java
View File

@ -48,7 +48,7 @@ public class ScanResultValue implements Comparable<ScanResultValue>
@JsonCreator
public ScanResultValue(
@Nullable @JsonProperty("segmentId") String segmentId,
@JsonProperty("segmentId") @Nullable String segmentId,
@JsonProperty("columns") List<String> columns,
@JsonProperty("events") Object events
)

2
processing/src/main/java/org/apache/druid/query/topn/TopNQueryBuilder.java
View File

@ -230,7 +230,7 @@ public class TopNQueryBuilder
public TopNQueryBuilder filters(String dimensionName, String value, String... values)
{
dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values), null);
dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values), null, null);
return this;
}

2
processing/src/main/java/org/apache/druid/segment/FilteredOffset.java
View File

@ -56,7 +56,7 @@ public final class FilteredOffset extends Offset
rowOffsetMatcherFactory
);
} else {
if (postFilter.supportsBitmapIndex(bitmapIndexSelector)) {
if (postFilter.shouldUseBitmapIndex(bitmapIndexSelector)) {
filterMatcher = rowOffsetMatcherFactory.makeRowOffsetMatcher(
postFilter.getBitmapIndex(bitmapIndexSelector)
);

28
processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java
View File

@ -19,6 +19,7 @@
package org.apache.druid.segment;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Sets;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.java.util.common.DateTimes;
@ -212,7 +213,7 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
{
if (filter != null) {
final boolean filterCanVectorize =
filter.supportsBitmapIndex(makeBitmapIndexSelector(virtualColumns))
filter.shouldUseBitmapIndex(makeBitmapIndexSelector(virtualColumns))
|| filter.canVectorizeMatcher();
if (!filterCanVectorize) {
@ -347,7 +348,8 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
return interval.overlap(dataInterval);
}
private ColumnSelectorBitmapIndexSelector makeBitmapIndexSelector(final VirtualColumns virtualColumns)
@VisibleForTesting
public ColumnSelectorBitmapIndexSelector makeBitmapIndexSelector(final VirtualColumns virtualColumns)
{
return new ColumnSelectorBitmapIndexSelector(
index.getBitmapFactoryForDimensions(),
@ -356,9 +358,10 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
);
}
private FilterAnalysis analyzeFilter(
@VisibleForTesting
public FilterAnalysis analyzeFilter(
@Nullable final Filter filter,
ColumnSelectorBitmapIndexSelector bitmapIndexSelector,
ColumnSelectorBitmapIndexSelector indexSelector,
@Nullable QueryMetrics queryMetrics
)
{
@ -389,7 +392,9 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
if (filter instanceof AndFilter) {
// If we get an AndFilter, we can split the subfilters across both filtering stages
for (Filter subfilter : ((AndFilter) filter).getFilters()) {
if (subfilter.supportsBitmapIndex(bitmapIndexSelector)) {
if (subfilter.supportsBitmapIndex(indexSelector) && subfilter.shouldUseBitmapIndex(indexSelector)) {
preFilters.add(subfilter);
} else {
postFilters.add(subfilter);
@ -397,7 +402,7 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
}
} else {
// If we get an OrFilter or a single filter, handle the filter in one stage
if (filter.supportsBitmapIndex(bitmapIndexSelector)) {
if (filter.supportsBitmapIndex(indexSelector) && filter.shouldUseBitmapIndex(indexSelector)) {
preFilters.add(filter);
} else {
postFilters.add(filter);
@ -411,15 +416,15 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
} else {
if (queryMetrics != null) {
BitmapResultFactory<?> bitmapResultFactory =
queryMetrics.makeBitmapResultFactory(bitmapIndexSelector.getBitmapFactory());
queryMetrics.makeBitmapResultFactory(indexSelector.getBitmapFactory());
long bitmapConstructionStartNs = System.nanoTime();
// Use AndFilter.getBitmapResult to intersect the preFilters to get its short-circuiting behavior.
preFilterBitmap = AndFilter.getBitmapIndex(bitmapIndexSelector, bitmapResultFactory, preFilters);
preFilterBitmap = AndFilter.getBitmapIndex(indexSelector, bitmapResultFactory, preFilters);
preFilteredRows = preFilterBitmap.size();
queryMetrics.reportBitmapConstructionTime(System.nanoTime() - bitmapConstructionStartNs);
} else {
BitmapResultFactory<?> bitmapResultFactory = new DefaultBitmapResultFactory(bitmapIndexSelector.getBitmapFactory());
preFilterBitmap = AndFilter.getBitmapIndex(bitmapIndexSelector, bitmapResultFactory, preFilters);
BitmapResultFactory<?> bitmapResultFactory = new DefaultBitmapResultFactory(indexSelector.getBitmapFactory());
preFilterBitmap = AndFilter.getBitmapIndex(indexSelector, bitmapResultFactory, preFilters);
}
}
@ -442,7 +447,8 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
return new FilterAnalysis(preFilterBitmap, postFilter);
}
private static class FilterAnalysis
@VisibleForTesting
public static class FilterAnalysis
{
private final Filter postFilter;
private final ImmutableBitmap preFilterBitmap;

26
processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java
View File

@ -34,7 +34,6 @@ import org.apache.druid.query.filter.vector.BaseVectorValueMatcher;
import org.apache.druid.query.filter.vector.ReadableVectorMatch;
import org.apache.druid.query.filter.vector.VectorValueMatcher;
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
@ -159,31 +158,6 @@ public class AndFilter implements BooleanFilter
return filters;
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
for (Filter filter : filters) {
if (!filter.supportsBitmapIndex(selector)) {
return false;
}
}
return true;
}
@Override
public boolean supportsSelectivityEstimation(
final ColumnSelector columnSelector,
final BitmapIndexSelector indexSelector
)
{
for (Filter filter : filters) {
if (!filter.supportsSelectivityEstimation(columnSelector, indexSelector)) {
return false;
}
}
return true;
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
{

15
processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java
View File

@ -34,6 +34,7 @@ import org.apache.druid.query.filter.DruidFloatPredicate;
import org.apache.druid.query.filter.DruidLongPredicate;
import org.apache.druid.query.filter.DruidPredicateFactory;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.FilterTuning;
import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.query.filter.vector.VectorValueMatcher;
import org.apache.druid.query.filter.vector.VectorValueMatcherColumnStrategizer;
@ -46,12 +47,14 @@ import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import java.util.Comparator;
import java.util.Set;
public class BoundFilter implements Filter
{
private final BoundDimFilter boundDimFilter;
private final Comparator<String> comparator;
private final ExtractionFn extractionFn;
private final FilterTuning filterTuning;
private final Supplier<DruidLongPredicate> longPredicateSupplier;
private final Supplier<DruidFloatPredicate> floatPredicateSupplier;
@ -65,6 +68,7 @@ public class BoundFilter implements Filter
this.longPredicateSupplier = boundDimFilter.getLongPredicateSupplier();
this.floatPredicateSupplier = boundDimFilter.getFloatPredicateSupplier();
this.doublePredicateSupplier = boundDimFilter.getDoublePredicateSupplier();
this.filterTuning = boundDimFilter.getFilterTuning();
}
@Override
@ -149,6 +153,11 @@ public class BoundFilter implements Filter
{
return selector.getBitmapIndex(boundDimFilter.getDimension()) != null;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return Filters.shouldUseBitmapIndex(this, selector, filterTuning);
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
@ -156,6 +165,12 @@ public class BoundFilter implements Filter
return Filters.supportsSelectivityEstimation(this, boundDimFilter.getDimension(), columnSelector, indexSelector);
}
@Override
public Set<String> getRequiredColumns()
{
return boundDimFilter.getRequiredColumns();
}
private static Pair<Integer, Integer> getStartEndIndexes(
final BoundDimFilter boundDimFilter,
final BitmapIndex bitmapIndex

14
processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java
View File

@ -37,6 +37,8 @@ import org.apache.druid.segment.DimensionHandlerUtils;
import javax.annotation.Nullable;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
/**
*/
@ -141,12 +143,24 @@ public class ColumnComparisonFilter implements Filter
return false;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return false;
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
{
return false;
}
@Override
public Set<String> getRequiredColumns()
{
return dimensions.stream().map(DimensionSpec::getDimension).collect(Collectors.toSet());
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
{

28
processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java
View File

@ -21,6 +21,7 @@ package org.apache.druid.segment.filter;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableSet;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.extraction.ExtractionFn;
@ -30,6 +31,7 @@ import org.apache.druid.query.filter.DruidFloatPredicate;
import org.apache.druid.query.filter.DruidLongPredicate;
import org.apache.druid.query.filter.DruidPredicateFactory;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.FilterTuning;
import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.query.filter.vector.VectorValueMatcher;
import org.apache.druid.query.filter.vector.VectorValueMatcherColumnStrategizer;
@ -38,6 +40,8 @@ import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.DimensionHandlerUtils;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import java.util.Set;
/**
*/
public class DimensionPredicateFilter implements Filter
@ -46,17 +50,29 @@ public class DimensionPredicateFilter implements Filter
private final DruidPredicateFactory predicateFactory;
private final String basePredicateString;
private final ExtractionFn extractionFn;
private final FilterTuning filterTuning;
public DimensionPredicateFilter(
final String dimension,
final DruidPredicateFactory predicateFactory,
final ExtractionFn extractionFn
)
{
this(dimension, predicateFactory, extractionFn, null);
}
public DimensionPredicateFilter(
final String dimension,
final DruidPredicateFactory predicateFactory,
final ExtractionFn extractionFn,
final FilterTuning filterTuning
)
{
Preconditions.checkNotNull(predicateFactory, "predicateFactory");
this.dimension = Preconditions.checkNotNull(dimension, "dimension");
this.basePredicateString = predicateFactory.toString();
this.extractionFn = extractionFn;
this.filterTuning = filterTuning;
if (extractionFn == null) {
this.predicateFactory = predicateFactory;
@ -120,12 +136,24 @@ public class DimensionPredicateFilter implements Filter
return true;
}
@Override
public Set<String> getRequiredColumns()
{
return ImmutableSet.of(dimension);
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return selector.getBitmapIndex(dimension) != null;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return Filters.shouldUseBitmapIndex(this, selector, filterTuning);
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
{

17
processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java
View File

@ -30,6 +30,7 @@ import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.expression.ExprUtils;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.FilterTuning;
import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.ColumnSelector;
@ -44,11 +45,13 @@ public class ExpressionFilter implements Filter
{
private final Supplier<Expr> expr;
private final Supplier<Set<String>> requiredBindings;
private final FilterTuning filterTuning;
public ExpressionFilter(final Supplier<Expr> expr)
public ExpressionFilter(final Supplier<Expr> expr, final FilterTuning filterTuning)
{
this.expr = expr;
this.requiredBindings = Suppliers.memoize(() -> expr.get().analyzeInputs().getRequiredBindings());
this.filterTuning = filterTuning;
}
@Override
@ -108,6 +111,12 @@ public class ExpressionFilter implements Filter
}
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return Filters.shouldUseBitmapIndex(this, selector, filterTuning);
}
@Override
public <T> T getBitmapResult(final BitmapIndexSelector selector, final BitmapResultFactory<T> bitmapResultFactory)
{
@ -151,4 +160,10 @@ public class ExpressionFilter implements Filter
// Selectivity estimation not supported.
throw new UnsupportedOperationException();
}
@Override
public Set<String> getRequiredColumns()
{
return requiredBindings.get();
}
}

30
processing/src/main/java/org/apache/druid/segment/filter/Filters.java
View File

@ -39,6 +39,7 @@ import org.apache.druid.query.filter.DimFilter;
import org.apache.druid.query.filter.DruidLongPredicate;
import org.apache.druid.query.filter.DruidPredicateFactory;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.FilterTuning;
import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.query.filter.ValueMatcherColumnSelectorStrategy;
import org.apache.druid.query.filter.ValueMatcherColumnSelectorStrategyFactory;
@ -651,4 +652,33 @@ public class Filters
generateAllCombinations(result, andList.subList(1, andList.size()), nonAndList);
}
}
/**
* This method provides a "standard" implementation of {@link Filter#shouldUseBitmapIndex(BitmapIndexSelector)} which takes
* a {@link Filter}, a {@link BitmapIndexSelector}, and {@link FilterTuning} to determine if:
* a) the filter supports bitmap indexes for all required columns
* b) the filter tuning specifies that it should use the index
* c) the cardinality of the column is above the minimum threshold and below the maximum threshold to use the index
*
* If all these things are true, {@link org.apache.druid.segment.QueryableIndexStorageAdapter} will utilize the
* indexes.
*/
public static boolean shouldUseBitmapIndex(
Filter filter,
BitmapIndexSelector indexSelector,
@Nullable FilterTuning filterTuning
)
{
final FilterTuning tuning = filterTuning != null ? filterTuning : FilterTuning.createDefault(filter, indexSelector);
if (filter.supportsBitmapIndex(indexSelector) && tuning.getUseBitmapIndex()) {
return filter.getRequiredColumns().stream().allMatch(column -> {
final BitmapIndex index = indexSelector.getBitmapIndex(column);
Preconditions.checkNotNull(index, "Column does not have a bitmap index");
final int cardinality = index.getCardinality();
return cardinality >= tuning.getMinCardinalityToUseBitmapIndex()
&& cardinality <= tuning.getMaxCardinalityToUseBitmapIndex();
});
}
return false;
}
}

19
processing/src/main/java/org/apache/druid/segment/filter/InFilter.java
View File

@ -21,6 +21,7 @@ package org.apache.druid.segment.filter;
import com.google.common.base.Predicate;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableSet;
import it.unimi.dsi.fastutil.ints.IntIterable;
import it.unimi.dsi.fastutil.ints.IntIterator;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
@ -32,6 +33,7 @@ import org.apache.druid.query.filter.DruidFloatPredicate;
import org.apache.druid.query.filter.DruidLongPredicate;
import org.apache.druid.query.filter.DruidPredicateFactory;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.FilterTuning;
import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.query.filter.vector.VectorValueMatcher;
import org.apache.druid.query.filter.vector.VectorValueMatcherColumnStrategizer;
@ -52,6 +54,7 @@ public class InFilter implements Filter
private final String dimension;
private final Set<String> values;
private final ExtractionFn extractionFn;
private final FilterTuning filterTuning;
private final Supplier<DruidLongPredicate> longPredicateSupplier;
private final Supplier<DruidFloatPredicate> floatPredicateSupplier;
private final Supplier<DruidDoublePredicate> doublePredicateSupplier;
@ -62,12 +65,14 @@ public class InFilter implements Filter
Supplier<DruidLongPredicate> longPredicateSupplier,
Supplier<DruidFloatPredicate> floatPredicateSupplier,
Supplier<DruidDoublePredicate> doublePredicateSupplier,
ExtractionFn extractionFn
ExtractionFn extractionFn,
FilterTuning filterTuning
)
{
this.dimension = dimension;
this.values = values;
this.extractionFn = extractionFn;
this.filterTuning = filterTuning;
this.longPredicateSupplier = longPredicateSupplier;
this.floatPredicateSupplier = floatPredicateSupplier;
this.doublePredicateSupplier = doublePredicateSupplier;
@ -162,12 +167,24 @@ public class InFilter implements Filter
return true;
}
@Override
public Set<String> getRequiredColumns()
{
return ImmutableSet.of(dimension);
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return selector.getBitmapIndex(dimension) != null;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return Filters.shouldUseBitmapIndex(this, selector, filterTuning);
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
{

21
processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java
View File

@ -20,27 +20,34 @@
package org.apache.druid.segment.filter;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableSet;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.FilterTuning;
import org.apache.druid.query.filter.JavaScriptDimFilter;
import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.mozilla.javascript.Context;
import java.util.Set;
public class JavaScriptFilter implements Filter
{
private final String dimension;
private final JavaScriptDimFilter.JavaScriptPredicateFactory predicateFactory;
private final FilterTuning filterTuning;
public JavaScriptFilter(
String dimension,
JavaScriptDimFilter.JavaScriptPredicateFactory predicate
JavaScriptDimFilter.JavaScriptPredicateFactory predicate,
FilterTuning filterTuning
)
{
this.dimension = dimension;
this.predicateFactory = predicate;
this.filterTuning = filterTuning;
}
@Override
@ -92,9 +99,21 @@ public class JavaScriptFilter implements Filter
return selector.getBitmapIndex(dimension) != null;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return Filters.shouldUseBitmapIndex(this, selector, filterTuning);
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
{
return Filters.supportsSelectivityEstimation(this, dimension, columnSelector, indexSelector);
}
@Override
public Set<String> getRequiredColumns()
{
return ImmutableSet.of(dimension);
}
}

20
processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java
View File

@ -20,6 +20,7 @@
package org.apache.druid.segment.filter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import it.unimi.dsi.fastutil.ints.IntIterable;
import it.unimi.dsi.fastutil.ints.IntIterator;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
@ -28,6 +29,7 @@ import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.FilterTuning;
import org.apache.druid.query.filter.LikeDimFilter;
import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.query.filter.vector.VectorValueMatcher;
@ -43,22 +45,26 @@ import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.NoSuchElementException;
import java.util.Set;
public class LikeFilter implements Filter
{
private final String dimension;
private final ExtractionFn extractionFn;
private final LikeDimFilter.LikeMatcher likeMatcher;
private final FilterTuning filterTuning;
public LikeFilter(
final String dimension,
final ExtractionFn extractionFn,
final LikeDimFilter.LikeMatcher likeMatcher
final LikeDimFilter.LikeMatcher likeMatcher,
final FilterTuning filterTuning
)
{
this.dimension = dimension;
this.extractionFn = extractionFn;
this.likeMatcher = likeMatcher;
this.filterTuning = filterTuning;
}
@Override
@ -95,12 +101,24 @@ public class LikeFilter implements Filter
return true;
}
@Override
public Set<String> getRequiredColumns()
{
return ImmutableSet.of(dimension);
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return selector.getBitmapIndex(dimension) != null;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return Filters.shouldUseBitmapIndex(this, selector, filterTuning);
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
{

14
processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java
View File

@ -32,6 +32,8 @@ import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import java.util.Set;
/**
*/
public class NotFilter implements Filter
@ -103,12 +105,24 @@ public class NotFilter implements Filter
return baseFilter.canVectorizeMatcher();
}
@Override
public Set<String> getRequiredColumns()
{
return baseFilter.getRequiredColumns();
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return baseFilter.supportsBitmapIndex(selector);
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return baseFilter.shouldUseBitmapIndex(selector);
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
{

23
processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java
View File

@ -34,7 +34,6 @@ import org.apache.druid.query.filter.vector.ReadableVectorMatch;
import org.apache.druid.query.filter.vector.VectorMatch;
import org.apache.druid.query.filter.vector.VectorValueMatcher;
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
@ -133,28 +132,6 @@ public class OrFilter implements BooleanFilter
return filters;
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
for (Filter filter : filters) {
if (!filter.supportsBitmapIndex(selector)) {
return false;
}
}
return true;
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
{
for (Filter filter : filters) {
if (!filter.supportsSelectivityEstimation(columnSelector, indexSelector)) {
return false;
}
}
return true;
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
{

7
processing/src/main/java/org/apache/druid/segment/filter/RegexFilter.java
View File

@ -25,6 +25,7 @@ import org.apache.druid.query.filter.DruidDoublePredicate;
import org.apache.druid.query.filter.DruidFloatPredicate;
import org.apache.druid.query.filter.DruidLongPredicate;
import org.apache.druid.query.filter.DruidPredicateFactory;
import org.apache.druid.query.filter.FilterTuning;
import java.util.regex.Pattern;
@ -35,7 +36,8 @@ public class RegexFilter extends DimensionPredicateFilter
public RegexFilter(
final String dimension,
final Pattern pattern,
final ExtractionFn extractionFn
final ExtractionFn extractionFn,
final FilterTuning filterTuning
)
{
super(
@ -74,7 +76,8 @@ public class RegexFilter extends DimensionPredicateFilter
'}';
}
},
extractionFn
extractionFn,
filterTuning
);
}
}

7
processing/src/main/java/org/apache/druid/segment/filter/SearchQueryFilter.java
View File

@ -27,6 +27,7 @@ import org.apache.druid.query.filter.DruidDoublePredicate;
import org.apache.druid.query.filter.DruidFloatPredicate;
import org.apache.druid.query.filter.DruidLongPredicate;
import org.apache.druid.query.filter.DruidPredicateFactory;
import org.apache.druid.query.filter.FilterTuning;
import org.apache.druid.query.search.SearchQuerySpec;
/**
@ -37,7 +38,8 @@ public class SearchQueryFilter extends DimensionPredicateFilter
public SearchQueryFilter(
@JsonProperty("dimension") final String dimension,
@JsonProperty("query") final SearchQuerySpec query,
@JsonProperty("extractionFn") final ExtractionFn extractionFn
@JsonProperty("extractionFn") final ExtractionFn extractionFn,
@JsonProperty("filterTuning") final FilterTuning filterTuning
)
{
super(
@ -68,7 +70,8 @@ public class SearchQueryFilter extends DimensionPredicateFilter
return input -> query.accept(String.valueOf(input));
}
},
extractionFn
extractionFn,
filterTuning
);
}
}

29
processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java
View File

@ -19,10 +19,12 @@
package org.apache.druid.segment.filter;
import com.google.common.collect.ImmutableSet;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.FilterTuning;
import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.query.filter.vector.VectorValueMatcher;
import org.apache.druid.query.filter.vector.VectorValueMatcherColumnStrategizer;
@ -31,20 +33,35 @@ import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.DimensionHandlerUtils;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import javax.annotation.Nullable;
import java.util.Set;
/**
*/
public class SelectorFilter implements Filter
{
private final String dimension;
private final String value;
@Nullable
private final FilterTuning filterTuning;
public SelectorFilter(
String dimension,
String value
)
{
this(dimension, value, null);
}
public SelectorFilter(
String dimension,
String value,
@Nullable FilterTuning filterTuning
)
{
this.dimension = dimension;
this.value = value;
this.filterTuning = filterTuning;
}
@Override
@ -75,6 +92,12 @@ public class SelectorFilter implements Filter
return selector.getBitmapIndex(dimension) != null;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return Filters.shouldUseBitmapIndex(this, selector, filterTuning);
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
{
@ -93,6 +116,12 @@ public class SelectorFilter implements Filter
return true;
}
@Override
public Set<String> getRequiredColumns()
{
return ImmutableSet.of(dimension);
}
@Override
public String toString()
{

36
processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java
View File

@ -21,6 +21,7 @@ package org.apache.druid.segment.filter;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableSet;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.collections.spatial.search.Bound;
import org.apache.druid.query.BitmapResultFactory;
@ -30,25 +31,31 @@ import org.apache.druid.query.filter.DruidFloatPredicate;
import org.apache.druid.query.filter.DruidLongPredicate;
import org.apache.druid.query.filter.DruidPredicateFactory;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.FilterTuning;
import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.incremental.SpatialDimensionRowTransformer;
import java.util.Set;
/**
*/
public class SpatialFilter implements Filter
{
private final String dimension;
private final Bound bound;
private final FilterTuning filterTuning;
public SpatialFilter(
String dimension,
Bound bound
Bound bound,
FilterTuning filterTuning
)
{
this.dimension = Preconditions.checkNotNull(dimension, "dimension");
this.bound = Preconditions.checkNotNull(bound, "bound");
this.filterTuning = filterTuning;
}
@Override
@ -69,17 +76,12 @@ public class SpatialFilter implements Filter
@Override
public Predicate<String> makeStringPredicate()
{
return new Predicate<String>()
{
@Override
public boolean apply(String input)
{
if (input == null) {
return false;
}
final float[] coordinate = SpatialDimensionRowTransformer.decode(input);
return bound.contains(coordinate);
return input -> {
if (input == null) {
return false;
}
final float[] coordinate = SpatialDimensionRowTransformer.decode(input);
return bound.contains(coordinate);
};
}
@ -113,12 +115,24 @@ public class SpatialFilter implements Filter
return selector.getBitmapIndex(dimension) != null;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return Filters.shouldUseBitmapIndex(this, selector, filterTuning);
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
{
return false;
}
@Override
public Set<String> getRequiredColumns()
{
return ImmutableSet.of(dimension);
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
{

15
processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java
View File

@ -26,6 +26,9 @@ import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import java.util.Collections;
import java.util.Set;
/**
*/
public class TrueFilter implements Filter
@ -52,12 +55,24 @@ public class TrueFilter implements Filter
return true;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return true;
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
{
return true;
}
@Override
public Set<String> getRequiredColumns()
{
return Collections.emptySet();
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
{

2
processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java
View File

@ -98,7 +98,7 @@ public class DictionaryEncodedColumnPartSerde implements ColumnPartSerde
@JsonCreator
public static DictionaryEncodedColumnPartSerde createDeserializer(
@Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory,
@JsonProperty("bitmapSerdeFactory") @Nullable BitmapSerdeFactory bitmapSerdeFactory,
@NotNull @JsonProperty("byteOrder") ByteOrder byteOrder
)
{

2
processing/src/main/java/org/apache/druid/segment/serde/DoubleNumericColumnPartSerdeV2.java
View File

@ -42,7 +42,7 @@ public class DoubleNumericColumnPartSerdeV2 implements ColumnPartSerde
@JsonCreator
public static DoubleNumericColumnPartSerdeV2 getDoubleGenericColumnPartSerde(
@JsonProperty("byteOrder") ByteOrder byteOrder,
@Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory
@JsonProperty("bitmapSerdeFactory") @Nullable BitmapSerdeFactory bitmapSerdeFactory
)
{
return new DoubleNumericColumnPartSerdeV2(

2
processing/src/main/java/org/apache/druid/segment/serde/FloatNumericColumnPartSerdeV2.java
View File

@ -40,7 +40,7 @@ public class FloatNumericColumnPartSerdeV2 implements ColumnPartSerde
@JsonCreator
public static FloatNumericColumnPartSerdeV2 createDeserializer(
@JsonProperty("byteOrder") ByteOrder byteOrder,
@Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory
@JsonProperty("bitmapSerdeFactory") @Nullable BitmapSerdeFactory bitmapSerdeFactory
)
{
return new FloatNumericColumnPartSerdeV2(

2
processing/src/main/java/org/apache/druid/segment/serde/LongNumericColumnPartSerdeV2.java
View File

@ -40,7 +40,7 @@ public class LongNumericColumnPartSerdeV2 implements ColumnPartSerde
@JsonCreator
public static LongNumericColumnPartSerdeV2 createDeserializer(
@JsonProperty("byteOrder") ByteOrder byteOrder,
@Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory
@JsonProperty("bitmapSerdeFactory") @Nullable BitmapSerdeFactory bitmapSerdeFactory
)
{
return new LongNumericColumnPartSerdeV2(

8
processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java
View File

@ -37,7 +37,7 @@ public class InDimFilterSerDesrTest
{
private static ObjectMapper mapper;
private final String actualInFilter =
private final String serializedFilter =
"{\"type\":\"in\",\"dimension\":\"dimTest\",\"values\":[\"bad\",\"good\"],\"extractionFn\":null}";
@Before
@ -50,7 +50,7 @@ public class InDimFilterSerDesrTest
@Test
public void testDeserialization() throws IOException
{
final InDimFilter actualInDimFilter = mapper.readerFor(DimFilter.class).readValue(actualInFilter);
final InDimFilter actualInDimFilter = mapper.readerFor(DimFilter.class).readValue(serializedFilter);
final InDimFilter expectedInDimFilter = new InDimFilter("dimTest", Arrays.asList("good", "bad"), null);
Assert.assertEquals(expectedInDimFilter, actualInDimFilter);
}
@ -59,8 +59,8 @@ public class InDimFilterSerDesrTest
public void testSerialization() throws IOException
{
final InDimFilter dimInFilter = new InDimFilter("dimTest", Arrays.asList("good", "bad"), null);
final String expectedInFilter = mapper.writeValueAsString(dimInFilter);
Assert.assertEquals(expectedInFilter, actualInFilter);
final String actualSerializedFilter = mapper.writeValueAsString(dimInFilter);
Assert.assertEquals(serializedFilter, actualSerializedFilter);
}
@Test

28
processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
View File

@ -81,13 +81,15 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
public abstract class BaseFilterTest
{
private static final VirtualColumns VIRTUAL_COLUMNS = VirtualColumns.create(
static final VirtualColumns VIRTUAL_COLUMNS = VirtualColumns.create(
ImmutableList.of(
new ExpressionVirtualColumn("expr", "1.0 + 0.1", ValueType.FLOAT, TestExprMacroTable.INSTANCE),
new ExpressionVirtualColumn("exprDouble", "1.0 + 1.1", ValueType.DOUBLE, TestExprMacroTable.INSTANCE),
@ -396,12 +398,24 @@ public abstract class BaseFilterTest
return false;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return false;
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
{
return false;
}
@Override
public Set<String> getRequiredColumns()
{
return Collections.emptySet();
}
@Override
public double estimateSelectivity(BitmapIndexSelector indexSelector)
{
@ -458,6 +472,12 @@ public abstract class BaseFilterTest
return false;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return false;
}
@Override
public VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory factory)
{
@ -470,6 +490,12 @@ public abstract class BaseFilterTest
return theFilter.canVectorizeMatcher();
}
@Override
public Set<String> getRequiredColumns()
{
return null;
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
{

2
processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java
View File

@ -269,6 +269,6 @@ public class ExpressionFilterTest extends BaseFilterTest
private static ExpressionDimFilter edf(final String expression)
{
return new ExpressionDimFilter(expression, TestExprMacroTable.INSTANCE);
return new ExpressionDimFilter(expression, null, TestExprMacroTable.INSTANCE);
}
}

92
processing/src/test/java/org/apache/druid/segment/filter/FilterPartitionTest.java
View File

@ -43,9 +43,12 @@ import org.apache.druid.query.filter.DruidFloatPredicate;
import org.apache.druid.query.filter.DruidLongPredicate;
import org.apache.druid.query.filter.DruidPredicateFactory;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.FilterTuning;
import org.apache.druid.query.filter.OrDimFilter;
import org.apache.druid.query.filter.SelectorDimFilter;
import org.apache.druid.segment.ColumnSelectorBitmapIndexSelector;
import org.apache.druid.segment.IndexBuilder;
import org.apache.druid.segment.QueryableIndexStorageAdapter;
import org.apache.druid.segment.StorageAdapter;
import org.junit.AfterClass;
import org.junit.Assert;
@ -72,6 +75,15 @@ public class FilterPartitionTest extends BaseFilterTest
super(dimension, value);
}
public NoBitmapSelectorFilter(
String dimension,
String value,
FilterTuning filterTuning
)
{
super(dimension, value, filterTuning);
}
@Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
@ -727,4 +739,84 @@ public class FilterPartitionTest extends BaseFilterTest
ImmutableList.of("2", "3", "4", "6", "7", "9")
);
}
@Test
public void testAnalyze()
{
if (!(adapter instanceof QueryableIndexStorageAdapter)) {
return;
}
QueryableIndexStorageAdapter storageAdapter = (QueryableIndexStorageAdapter) adapter;
final ColumnSelectorBitmapIndexSelector bitmapIndexSelector = storageAdapter.makeBitmapIndexSelector(BaseFilterTest.VIRTUAL_COLUMNS);
// has bitmap index, will use it by default
Filter normalFilter = new SelectorFilter("dim1", "HELLO");
QueryableIndexStorageAdapter.FilterAnalysis filterAnalysisNormal =
storageAdapter.analyzeFilter(normalFilter, bitmapIndexSelector, null);
Assert.assertTrue(filterAnalysisNormal.getPreFilterBitmap() != null);
Assert.assertTrue(filterAnalysisNormal.getPostFilter() == null);
// no bitmap index, should be a post filter
Filter noBitmapFilter = new NoBitmapSelectorFilter("dim1", "HELLO");
QueryableIndexStorageAdapter.FilterAnalysis noBitmapFilterAnalysis =
storageAdapter.analyzeFilter(noBitmapFilter, bitmapIndexSelector, null);
Assert.assertTrue(noBitmapFilterAnalysis.getPreFilterBitmap() == null);
Assert.assertTrue(noBitmapFilterAnalysis.getPostFilter() != null);
// this column has a bitmap index, but is forced to not use it
Filter bitmapFilterWithForceNoIndexTuning = new SelectorFilter(
"dim1",
"HELLO",
new FilterTuning(false, null, null)
);
QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithForceNoIndexTuningAnalysis =
storageAdapter.analyzeFilter(bitmapFilterWithForceNoIndexTuning, bitmapIndexSelector, null);
Assert.assertTrue(bitmapFilterWithForceNoIndexTuningAnalysis.getPreFilterBitmap() == null);
Assert.assertTrue(bitmapFilterWithForceNoIndexTuningAnalysis.getPostFilter() != null);
// this max cardinality is too low to use bitmap index
Filter bitmapFilterWithCardinalityMax = new SelectorFilter(
"dim1",
"HELLO",
new FilterTuning(true, 0, 3)
);
QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithCardinalityMaxAnalysis =
storageAdapter.analyzeFilter(bitmapFilterWithCardinalityMax, bitmapIndexSelector, null);
Assert.assertTrue(bitmapFilterWithCardinalityMaxAnalysis.getPreFilterBitmap() == null);
Assert.assertTrue(bitmapFilterWithCardinalityMaxAnalysis.getPostFilter() != null);
// this max cardinality is high enough that we can still use bitmap index
Filter bitmapFilterWithCardinalityMax2 = new SelectorFilter(
"dim1",
"HELLO",
new FilterTuning(true, 0, 1000)
);
QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithCardinalityMax2Analysis =
storageAdapter.analyzeFilter(bitmapFilterWithCardinalityMax2, bitmapIndexSelector, null);
Assert.assertTrue(bitmapFilterWithCardinalityMax2Analysis.getPreFilterBitmap() != null);
Assert.assertTrue(bitmapFilterWithCardinalityMax2Analysis.getPostFilter() == null);
// this min cardinality is too high, will not use bitmap index
Filter bitmapFilterWithCardinalityMin = new SelectorFilter(
"dim1",
"HELLO",
new FilterTuning(true, 1000, null)
);
QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithCardinalityMinAnalysis =
storageAdapter.analyzeFilter(bitmapFilterWithCardinalityMin, bitmapIndexSelector, null);
Assert.assertTrue(bitmapFilterWithCardinalityMinAnalysis.getPreFilterBitmap() == null);
Assert.assertTrue(bitmapFilterWithCardinalityMinAnalysis.getPostFilter() != null);
// cannot force using bitmap if there are no bitmaps
Filter noBitmapFilterWithForceUse = new NoBitmapSelectorFilter(
"dim1",
"HELLO",
new FilterTuning(true, null, null)
);
QueryableIndexStorageAdapter.FilterAnalysis noBitmapFilterWithForceUseAnalysis =
storageAdapter.analyzeFilter(noBitmapFilterWithForceUse, bitmapIndexSelector, null);
Assert.assertTrue(noBitmapFilterWithForceUseAnalysis.getPreFilterBitmap() == null);
Assert.assertTrue(noBitmapFilterWithForceUseAnalysis.getPostFilter() != null);
}
}

13
processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java
View File

@ -77,6 +77,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
/**
@ -693,12 +694,24 @@ public class IncrementalIndexStorageAdapterTest
return true;
}
@Override
public boolean shouldUseBitmapIndex(BitmapIndexSelector selector)
{
return true;
}
@Override
public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector)
{
return true;
}
@Override
public Set<String> getRequiredColumns()
{
return Collections.emptySet();
}
private class DictionaryRaceTestFilterDruidPredicateFactory implements DruidPredicateFactory
{
@Override

4
server/src/main/java/org/apache/druid/client/indexing/ClientCompactQuery.java
View File

@ -46,8 +46,8 @@ public class ClientCompactQuery implements ClientQuery
@JsonCreator
public ClientCompactQuery(
@JsonProperty("dataSource") String dataSource,
@Nullable @JsonProperty("interval") final Interval interval,
@Nullable @JsonProperty("segments") final List<DataSegment> segments,
@JsonProperty("interval") @Nullable final Interval interval,
@JsonProperty("segments") @Nullable final List<DataSegment> segments,
@JsonProperty("targetCompactionSizeBytes") @Nullable Long targetCompactionSizeBytes,
@JsonProperty("tuningConfig") ClientCompactQueryTuningConfig tuningConfig,
@JsonProperty("context") Map<String, Object> context

2
server/src/main/java/org/apache/druid/client/indexing/IndexingWorkerInfo.java
View File

@ -46,7 +46,7 @@ public class IndexingWorkerInfo
@JsonProperty("availabilityGroups") Set<String> availabilityGroups,
@JsonProperty("runningTasks") Collection<String> runningTasks,
@JsonProperty("lastCompletedTaskTime") DateTime lastCompletedTaskTime,
@Nullable @JsonProperty("blacklistedUntil") DateTime blacklistedUntil
@JsonProperty("blacklistedUntil") @Nullable DateTime blacklistedUntil
)
{
this.worker = worker;

6
server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java
View File

@ -59,9 +59,9 @@ public class NoopSupervisorSpec implements SupervisorSpec
@JsonCreator
public NoopSupervisorSpec(
@Nullable @JsonProperty("id") String id,
@Nullable @JsonProperty("dataSources") List<String> datasources,
@Nullable @JsonProperty("suspended") Boolean suspended
@JsonProperty("id") @Nullable String id,
@JsonProperty("dataSources") @Nullable List<String> datasources,
@JsonProperty("suspended") @Nullable Boolean suspended
)
{
this.id = id;

27
sql/src/main/java/org/apache/druid/sql/calcite/filtration/Bounds.java
View File

@ -53,7 +53,8 @@ public class Bounds
false,
null,
bound.getExtractionFn(),
bound.getOrdering()
bound.getOrdering(),
bound.getFilterTuning()
);
} else {
// bound.getLower() != null
@ -65,7 +66,8 @@ public class Bounds
!bound.isLowerStrict(),
null,
bound.getExtractionFn(),
bound.getOrdering()
bound.getOrdering(),
bound.getFilterTuning()
);
}
}
@ -113,7 +115,8 @@ public class Bounds
range.hasUpperBound() && range.upperBoundType() == BoundType.OPEN,
null,
boundRefKey.getExtractionFn(),
boundRefKey.getComparator()
boundRefKey.getComparator(),
null
);
}
@ -127,7 +130,8 @@ public class Bounds
false,
null,
boundRefKey.getExtractionFn(),
boundRefKey.getComparator()
boundRefKey.getComparator(),
null
);
}
@ -141,7 +145,8 @@ public class Bounds
false,
null,
boundRefKey.getExtractionFn(),
boundRefKey.getComparator()
boundRefKey.getComparator(),
null
);
}
@ -155,7 +160,8 @@ public class Bounds
false,
null,
boundRefKey.getExtractionFn(),
boundRefKey.getComparator()
boundRefKey.getComparator(),
null
);
}
@ -169,7 +175,8 @@ public class Bounds
true,
null,
boundRefKey.getExtractionFn(),
boundRefKey.getComparator()
boundRefKey.getComparator(),
null
);
}
@ -183,7 +190,8 @@ public class Bounds
false,
null,
boundRefKey.getExtractionFn(),
boundRefKey.getComparator()
boundRefKey.getComparator(),
null
);
}
@ -202,7 +210,8 @@ public class Bounds
true,
null,
boundRefKey.getExtractionFn(),
boundRefKey.getComparator()
boundRefKey.getComparator(),
null
);
}
}

2
sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertSelectorsToIns.java
View File

@ -89,7 +89,7 @@ public class ConvertSelectorsToIns extends BottomUpTransform
}
}
children.add(new InDimFilter(entry.getKey().getDimension(), values, entry.getKey().getExtractionFn()));
children.add(new InDimFilter(entry.getKey().getDimension(), values, entry.getKey().getExtractionFn(), null));
}
}