diff --git a/processing/src/main/java/io/druid/query/Druids.java b/processing/src/main/java/io/druid/query/Druids.java index 920c8da8635..d7a0870e2fd 100644 --- a/processing/src/main/java/io/druid/query/Druids.java +++ b/processing/src/main/java/io/druid/query/Druids.java @@ -32,6 +32,7 @@ import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.AndDimFilter; import io.druid.query.filter.DimFilter; +import io.druid.query.filter.InDimFilter; import io.druid.query.filter.NoopDimFilter; import io.druid.query.filter.NotDimFilter; import io.druid.query.filter.OrDimFilter; @@ -464,11 +465,7 @@ public class Druids public TimeseriesQueryBuilder filters(String dimensionName, String value, String... values) { - List fields = Lists.newArrayList(new SelectorDimFilter(dimensionName, value)); - for (String val : values) { - fields.add(new SelectorDimFilter(dimensionName, val)); - } - dimFilter = new OrDimFilter(fields); + dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values)); return this; } @@ -624,11 +621,7 @@ public class Druids public SearchQueryBuilder filters(String dimensionName, String value, String... values) { - List fields = Lists.newArrayList(new SelectorDimFilter(dimensionName, value)); - for (String val : values) { - fields.add(new SelectorDimFilter(dimensionName, val)); - } - dimFilter = new OrDimFilter(fields); + dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values)); return this; } @@ -1172,11 +1165,7 @@ public class Druids public SelectQueryBuilder filters(String dimensionName, String value, String... values) { - List fields = Lists.newArrayList(new SelectorDimFilter(dimensionName, value)); - for (String val : values) { - fields.add(new SelectorDimFilter(dimensionName, val)); - } - dimFilter = new OrDimFilter(fields); + dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values)); return this; } diff --git a/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java b/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java index 23e5c05827c..81bf20e4c9b 100644 --- a/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java @@ -21,9 +21,7 @@ package io.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; import com.metamx.common.StringUtils; import io.druid.query.extraction.ExtractionFn; import io.druid.query.lookup.LookupExtractionFn; @@ -104,14 +102,7 @@ public class ExtractionDimFilter implements DimFilter final List keys = lookup.unapply(this.getValue()); final String dimensionName = this.getDimension(); if (!keys.isEmpty()) { - return new OrDimFilter(Lists.transform(keys, new Function() - { - @Override - public DimFilter apply(String input) - { - return new SelectorDimFilter(dimensionName, input); - } - })); + return new InDimFilter(dimensionName, keys); } } return this; diff --git a/processing/src/main/java/io/druid/query/filter/InDimFilter.java b/processing/src/main/java/io/druid/query/filter/InDimFilter.java index 675b11ba9fa..e9167f5a939 100644 --- a/processing/src/main/java/io/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/InDimFilter.java @@ -24,14 +24,14 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.metamx.common.StringUtils; -import io.druid.segment.filter.OrFilter; -import io.druid.segment.filter.SelectorFilter; +import io.druid.segment.filter.InFilter; import java.nio.ByteBuffer; -import java.util.Collections; import java.util.List; public class InDimFilter implements DimFilter @@ -43,7 +43,20 @@ public class InDimFilter implements DimFilter public InDimFilter(@JsonProperty("dimension") String dimension, @JsonProperty("values") List values) { Preconditions.checkNotNull(dimension, "dimension can not be null"); - this.values = (values == null) ? Collections.emptyList() : values; + Preconditions.checkArgument(values != null && !values.isEmpty(), "values can not be null or empty"); + this.values = Lists.newArrayList( + Iterables.transform( + values, new Function() + { + @Override + public String apply(String input) + { + return Strings.nullToEmpty(input); + } + + } + ) + ); this.dimension = dimension; } @@ -92,21 +105,7 @@ public class InDimFilter implements DimFilter @Override public Filter toFilter() { - final List selectorFilters = ImmutableList.copyOf( - Iterables.transform( - values, - new Function() - { - @Override - public Filter apply(String input) - { - return new SelectorFilter(dimension, input); - } - } - ) - ); - - return new OrFilter(selectorFilters); + return new InFilter(dimension, ImmutableSet.copyOf(values)); } @Override diff --git a/processing/src/main/java/io/druid/query/topn/TopNQueryBuilder.java b/processing/src/main/java/io/druid/query/topn/TopNQueryBuilder.java index 30f5a1ccff9..525a2b92acc 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNQueryBuilder.java +++ b/processing/src/main/java/io/druid/query/topn/TopNQueryBuilder.java @@ -28,7 +28,7 @@ import io.druid.query.aggregation.PostAggregator; import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.DimFilter; -import io.druid.query.filter.OrDimFilter; +import io.druid.query.filter.InDimFilter; import io.druid.query.filter.SelectorDimFilter; import io.druid.query.spec.LegacySegmentSpec; import io.druid.query.spec.QuerySegmentSpec; @@ -252,11 +252,7 @@ public class TopNQueryBuilder public TopNQueryBuilder filters(String dimensionName, String value, String... values) { - List fields = Lists.newArrayList(new SelectorDimFilter(dimensionName, value)); - for (String val : values) { - fields.add(new SelectorDimFilter(dimensionName, val)); - } - dimFilter = new OrDimFilter(fields); + dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values)); return this; } diff --git a/processing/src/main/java/io/druid/segment/filter/InFilter.java b/processing/src/main/java/io/druid/segment/filter/InFilter.java new file mode 100644 index 00000000000..e637cbae826 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/filter/InFilter.java @@ -0,0 +1,80 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.filter; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.base.Strings; +import com.google.common.collect.Iterables; +import com.metamx.collections.bitmap.BitmapFactory; +import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.query.filter.BitmapIndexSelector; +import io.druid.query.filter.Filter; +import io.druid.query.filter.ValueMatcher; +import io.druid.query.filter.ValueMatcherFactory; + +import javax.annotation.Nullable; +import java.util.Set; + +/** + */ +public class InFilter implements Filter +{ + private final String dimension; + private final Set values; + + public InFilter(String dimension, Set values) + { + this.dimension = dimension; + this.values = values; + } + + @Override + public ImmutableBitmap getBitmapIndex(final BitmapIndexSelector selector) + { + return selector.getBitmapFactory().union( + Iterables.transform( + values, new Function() + { + @Override + public ImmutableBitmap apply(String value) + { + return selector.getBitmapIndex(dimension, value); + } + } + ) + ); + } + + @Override + public ValueMatcher makeMatcher(ValueMatcherFactory factory) + { + return factory.makeValueMatcher( + dimension, new Predicate() + { + @Override + public boolean apply(String input) + { + return values.contains(Strings.nullToEmpty(input)); + } + } + ); + } +} diff --git a/processing/src/test/java/io/druid/segment/filter/InFilterTest.java b/processing/src/test/java/io/druid/segment/filter/InFilterTest.java new file mode 100644 index 00000000000..99e0d99d0fc --- /dev/null +++ b/processing/src/test/java/io/druid/segment/filter/InFilterTest.java @@ -0,0 +1,161 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.filter; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.metamx.common.Pair; +import io.druid.data.input.InputRow; +import io.druid.data.input.impl.DimensionsSpec; +import io.druid.data.input.impl.InputRowParser; +import io.druid.data.input.impl.MapInputRowParser; +import io.druid.data.input.impl.TimeAndDimsParseSpec; +import io.druid.data.input.impl.TimestampSpec; +import io.druid.query.filter.DimFilter; +import io.druid.query.filter.Filter; +import io.druid.query.filter.InDimFilter; +import io.druid.segment.IndexBuilder; +import io.druid.segment.StorageAdapter; +import org.joda.time.DateTime; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +@RunWith(Parameterized.class) +public class InFilterTest extends BaseFilterTest +{ + private static final String TIMESTAMP_COLUMN = "timestamp"; + + private static final InputRowParser> PARSER = new MapInputRowParser( + new TimeAndDimsParseSpec( + new TimestampSpec(TIMESTAMP_COLUMN, "iso", new DateTime("2000")), + new DimensionsSpec(null, null, null) + ) + ); + + private static final List ROWS = ImmutableList.of( + PARSER.parse(ImmutableMap.of("dim0", "a", "dim1", "", "dim2", ImmutableList.of("a", "b"))), + PARSER.parse(ImmutableMap.of("dim0", "b", "dim1", "10", "dim2", ImmutableList.of())), + PARSER.parse(ImmutableMap.of("dim0", "c", "dim1", "2", "dim2", ImmutableList.of(""))), + PARSER.parse(ImmutableMap.of("dim0", "d", "dim1", "1", "dim2", ImmutableList.of("a"))), + PARSER.parse(ImmutableMap.of("dim0", "e", "dim1", "def", "dim2", ImmutableList.of("c"))), + PARSER.parse(ImmutableMap.of("dim0", "f", "dim1", "abc")) + ); + + @Parameterized.Parameters(name = "{0}") + public static Collection constructorFeeder() throws IOException + { + return makeConstructors(); + } + + public InFilterTest( + String testName, + IndexBuilder indexBuilder, + Function> finisher + ) + { + super(ROWS, indexBuilder, finisher); + } + + @Before + public void setUp() throws IOException + { + final Pair pair = finisher.apply( + indexBuilder.tmpDir(temporaryFolder.newFolder()).add(ROWS) + ); + this.adapter = pair.lhs; + this.closeable = pair.rhs; + } + + @Test + public void testSingleValueStringColumnWithoutNulls() + { + Assert.assertEquals(ImmutableList.of(), select(toInFilter("dim0", null))); + Assert.assertEquals(ImmutableList.of(), select(toInFilter("dim0", "", ""))); + Assert.assertEquals(ImmutableList.of(0, 2), select(toInFilter("dim0", "a", "c"))); + Assert.assertEquals(ImmutableList.of(4), select(toInFilter("dim0", "e", "x"))); + } + + @Test + public void testSingleValueStringColumnWithNulls() + { + Assert.assertEquals(ImmutableList.of(0), select(toInFilter("dim1", null, ""))); + Assert.assertEquals(ImmutableList.of(0), select(toInFilter("dim1", ""))); + Assert.assertEquals(ImmutableList.of(0, 1, 5), select(toInFilter("dim1", null, "10", "abc"))); + Assert.assertEquals(ImmutableList.of(), select(toInFilter("dim1", "-1", "ab", "de"))); + } + + @Test + public void testMultiValueStringColumn() + { + Assert.assertEquals(ImmutableList.of(1, 2, 5), select(toInFilter("dim2", null))); + Assert.assertEquals(ImmutableList.of(1, 2, 5), select(toInFilter("dim2", "", (String)null))); + Assert.assertEquals(ImmutableList.of(0, 1, 2, 3, 5), select(toInFilter("dim2", null, "a"))); + Assert.assertEquals(ImmutableList.of(0, 1, 2, 5), select(toInFilter("dim2", null, "b"))); + Assert.assertEquals(ImmutableList.of(4), select(toInFilter("dim2", "c"))); + Assert.assertEquals(ImmutableList.of(), select(toInFilter("dim2", "d"))); + } + + @Test + public void testMissingColumn() + { + Assert.assertEquals(ImmutableList.of(0, 1, 2, 3, 4, 5), select(toInFilter("dim3", null, (String)null))); + Assert.assertEquals(ImmutableList.of(0, 1, 2, 3, 4, 5), select(toInFilter("dim3", ""))); + Assert.assertEquals(ImmutableList.of(0, 1, 2, 3, 4, 5), select(toInFilter("dim3", null, "a"))); + Assert.assertEquals(ImmutableList.of(), select(toInFilter("dim3", "a"))); + Assert.assertEquals(ImmutableList.of(), select(toInFilter("dim3", "b"))); + Assert.assertEquals(ImmutableList.of(), select(toInFilter("dim3", "c"))); + } + + private DimFilter toInFilter(String dim, String value, String... values) + { + return new InDimFilter(dim, Lists.asList(value, values)); + } + + private List select(final DimFilter filter) + { + return Lists.newArrayList( + Iterables.transform( + selectColumnValuesMatchingFilter(filter, "dim0"), + new Function() + { + @Override + public Integer apply(String input) + { + Preconditions.checkArgument(input.length() == 1); + return ((int) input.charAt(0)) - ((int) 'a'); + } + } + ) + ); + } +}