Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implement native in filter (Fix for #2577)

This commit is contained in:
navis.ryu 2016-03-02 15:38:01 +09:00
parent 595d359c3b
commit 108535fd07
6 changed files with 267 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
processing/src/main/java/io/druid/query/Druids.java
View File

@ -32,6 +32,7 @@ import io.druid.query.dimension.DefaultDimensionSpec;
import io.druid.query.dimension.DimensionSpec;
import io.druid.query.filter.AndDimFilter;
import io.druid.query.filter.DimFilter;
import io.druid.query.filter.InDimFilter;
import io.druid.query.filter.NoopDimFilter;
import io.druid.query.filter.NotDimFilter;
import io.druid.query.filter.OrDimFilter;
@ -464,11 +465,7 @@ public class Druids
public TimeseriesQueryBuilder filters(String dimensionName, String value, String... values)
{
List<DimFilter> fields = Lists.<DimFilter>newArrayList(new SelectorDimFilter(dimensionName, value));
for (String val : values) {
fields.add(new SelectorDimFilter(dimensionName, val));
}
dimFilter = new OrDimFilter(fields);
dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values));
return this;
}
@ -624,11 +621,7 @@ public class Druids
public SearchQueryBuilder filters(String dimensionName, String value, String... values)
{
List<DimFilter> fields = Lists.<DimFilter>newArrayList(new SelectorDimFilter(dimensionName, value));
for (String val : values) {
fields.add(new SelectorDimFilter(dimensionName, val));
}
dimFilter = new OrDimFilter(fields);
dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values));
return this;
}
@ -1172,11 +1165,7 @@ public class Druids
public SelectQueryBuilder filters(String dimensionName, String value, String... values)
{
List<DimFilter> fields = Lists.<DimFilter>newArrayList(new SelectorDimFilter(dimensionName, value));
for (String val : values) {
fields.add(new SelectorDimFilter(dimensionName, val));
}
dimFilter = new OrDimFilter(fields);
dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values));
return this;
}

11
processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java
View File

@ -21,9 +21,7 @@ package io.druid.query.filter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.metamx.common.StringUtils;
import io.druid.query.extraction.ExtractionFn;
import io.druid.query.lookup.LookupExtractionFn;
@ -104,14 +102,7 @@ public class ExtractionDimFilter implements DimFilter
final List<String> keys = lookup.unapply(this.getValue());
final String dimensionName = this.getDimension();
if (!keys.isEmpty()) {
return new OrDimFilter(Lists.transform(keys, new Function<String, DimFilter>()
{
@Override
public DimFilter apply(String input)
{
return new SelectorDimFilter(dimensionName, input);
}
}));
return new InDimFilter(dimensionName, keys);
}
}
return this;

39
processing/src/main/java/io/druid/query/filter/InDimFilter.java
View File

@ -24,14 +24,14 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.metamx.common.StringUtils;
import io.druid.segment.filter.OrFilter;
import io.druid.segment.filter.SelectorFilter;
import io.druid.segment.filter.InFilter;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
public class InDimFilter implements DimFilter
@ -43,7 +43,20 @@ public class InDimFilter implements DimFilter
public InDimFilter(@JsonProperty("dimension") String dimension, @JsonProperty("values") List<String> values)
{
Preconditions.checkNotNull(dimension, "dimension can not be null");
this.values = (values == null) ? Collections.<String>emptyList() : values;
Preconditions.checkArgument(values != null && !values.isEmpty(), "values can not be null or empty");
this.values = Lists.newArrayList(
Iterables.transform(
values, new Function<String, String>()
{
@Override
public String apply(String input)
{
return Strings.nullToEmpty(input);
}
}
)
);
this.dimension = dimension;
}
@ -92,21 +105,7 @@ public class InDimFilter implements DimFilter
@Override
public Filter toFilter()
{
final List<Filter> selectorFilters = ImmutableList.copyOf(
Iterables.transform(
values,
new Function<String, Filter>()
{
@Override
public Filter apply(String input)
{
return new SelectorFilter(dimension, input);
}
}
)
);
return new OrFilter(selectorFilters);
return new InFilter(dimension, ImmutableSet.copyOf(values));
}
@Override

8
processing/src/main/java/io/druid/query/topn/TopNQueryBuilder.java
View File

@ -28,7 +28,7 @@ import io.druid.query.aggregation.PostAggregator;
import io.druid.query.dimension.DefaultDimensionSpec;
import io.druid.query.dimension.DimensionSpec;
import io.druid.query.filter.DimFilter;
import io.druid.query.filter.OrDimFilter;
import io.druid.query.filter.InDimFilter;
import io.druid.query.filter.SelectorDimFilter;
import io.druid.query.spec.LegacySegmentSpec;
import io.druid.query.spec.QuerySegmentSpec;
@ -252,11 +252,7 @@ public class TopNQueryBuilder
public TopNQueryBuilder filters(String dimensionName, String value, String... values)
{
List<DimFilter> fields = Lists.<DimFilter>newArrayList(new SelectorDimFilter(dimensionName, value));
for (String val : values) {
fields.add(new SelectorDimFilter(dimensionName, val));
}
dimFilter = new OrDimFilter(fields);
dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values));
return this;
}

80
processing/src/main/java/io/druid/segment/filter/InFilter.java Normal file
View File

@ -0,0 +1,80 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.segment.filter;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.base.Strings;
import com.google.common.collect.Iterables;
import com.metamx.collections.bitmap.BitmapFactory;
import com.metamx.collections.bitmap.ImmutableBitmap;
import io.druid.query.filter.BitmapIndexSelector;
import io.druid.query.filter.Filter;
import io.druid.query.filter.ValueMatcher;
import io.druid.query.filter.ValueMatcherFactory;
import javax.annotation.Nullable;
import java.util.Set;
/**
*/
public class InFilter implements Filter
{
private final String dimension;
private final Set<String> values;
public InFilter(String dimension, Set<String> values)
{
this.dimension = dimension;
this.values = values;
}
@Override
public ImmutableBitmap getBitmapIndex(final BitmapIndexSelector selector)
{
return selector.getBitmapFactory().union(
Iterables.transform(
values, new Function<String, ImmutableBitmap>()
{
@Override
public ImmutableBitmap apply(String value)
{
return selector.getBitmapIndex(dimension, value);
}
}
)
);
}
@Override
public ValueMatcher makeMatcher(ValueMatcherFactory factory)
{
return factory.makeValueMatcher(
dimension, new Predicate<String>()
{
@Override
public boolean apply(String input)
{
return values.contains(Strings.nullToEmpty(input));
}
}
);
}
}

161
processing/src/test/java/io/druid/segment/filter/InFilterTest.java Normal file
View File

@ -0,0 +1,161 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.segment.filter;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.metamx.common.Pair;
import io.druid.data.input.InputRow;
import io.druid.data.input.impl.DimensionsSpec;
import io.druid.data.input.impl.InputRowParser;
import io.druid.data.input.impl.MapInputRowParser;
import io.druid.data.input.impl.TimeAndDimsParseSpec;
import io.druid.data.input.impl.TimestampSpec;
import io.druid.query.filter.DimFilter;
import io.druid.query.filter.Filter;
import io.druid.query.filter.InDimFilter;
import io.druid.segment.IndexBuilder;
import io.druid.segment.StorageAdapter;
import org.joda.time.DateTime;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.io.Closeable;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@RunWith(Parameterized.class)
public class InFilterTest extends BaseFilterTest
{
private static final String TIMESTAMP_COLUMN = "timestamp";
private static final InputRowParser<Map<String, Object>> PARSER = new MapInputRowParser(
new TimeAndDimsParseSpec(
new TimestampSpec(TIMESTAMP_COLUMN, "iso", new DateTime("2000")),
new DimensionsSpec(null, null, null)
)
);
private static final List<InputRow> ROWS = ImmutableList.of(
PARSER.parse(ImmutableMap.<String, Object>of("dim0", "a", "dim1", "", "dim2", ImmutableList.of("a", "b"))),
PARSER.parse(ImmutableMap.<String, Object>of("dim0", "b", "dim1", "10", "dim2", ImmutableList.of())),
PARSER.parse(ImmutableMap.<String, Object>of("dim0", "c", "dim1", "2", "dim2", ImmutableList.of(""))),
PARSER.parse(ImmutableMap.<String, Object>of("dim0", "d", "dim1", "1", "dim2", ImmutableList.of("a"))),
PARSER.parse(ImmutableMap.<String, Object>of("dim0", "e", "dim1", "def", "dim2", ImmutableList.of("c"))),
PARSER.parse(ImmutableMap.<String, Object>of("dim0", "f", "dim1", "abc"))
);
@Parameterized.Parameters(name = "{0}")
public static Collection<Object[]> constructorFeeder() throws IOException
{
return makeConstructors();
}
public InFilterTest(
String testName,
IndexBuilder indexBuilder,
Function<IndexBuilder, Pair<StorageAdapter, Closeable>> finisher
)
{
super(ROWS, indexBuilder, finisher);
}
@Before
public void setUp() throws IOException
{
final Pair<StorageAdapter, Closeable> pair = finisher.apply(
indexBuilder.tmpDir(temporaryFolder.newFolder()).add(ROWS)
);
this.adapter = pair.lhs;
this.closeable = pair.rhs;
}
@Test
public void testSingleValueStringColumnWithoutNulls()
{
Assert.assertEquals(ImmutableList.<Integer>of(), select(toInFilter("dim0", null)));
Assert.assertEquals(ImmutableList.<Integer>of(), select(toInFilter("dim0", "", "")));
Assert.assertEquals(ImmutableList.of(0, 2), select(toInFilter("dim0", "a", "c")));
Assert.assertEquals(ImmutableList.of(4), select(toInFilter("dim0", "e", "x")));
}
@Test
public void testSingleValueStringColumnWithNulls()
{
Assert.assertEquals(ImmutableList.of(0), select(toInFilter("dim1", null, "")));
Assert.assertEquals(ImmutableList.of(0), select(toInFilter("dim1", "")));
Assert.assertEquals(ImmutableList.of(0, 1, 5), select(toInFilter("dim1", null, "10", "abc")));
Assert.assertEquals(ImmutableList.<Integer>of(), select(toInFilter("dim1", "-1", "ab", "de")));
}
@Test
public void testMultiValueStringColumn()
{
Assert.assertEquals(ImmutableList.of(1, 2, 5), select(toInFilter("dim2", null)));
Assert.assertEquals(ImmutableList.of(1, 2, 5), select(toInFilter("dim2", "", (String)null)));
Assert.assertEquals(ImmutableList.of(0, 1, 2, 3, 5), select(toInFilter("dim2", null, "a")));
Assert.assertEquals(ImmutableList.of(0, 1, 2, 5), select(toInFilter("dim2", null, "b")));
Assert.assertEquals(ImmutableList.of(4), select(toInFilter("dim2", "c")));
Assert.assertEquals(ImmutableList.<Integer>of(), select(toInFilter("dim2", "d")));
}
@Test
public void testMissingColumn()
{
Assert.assertEquals(ImmutableList.of(0, 1, 2, 3, 4, 5), select(toInFilter("dim3", null, (String)null)));
Assert.assertEquals(ImmutableList.of(0, 1, 2, 3, 4, 5), select(toInFilter("dim3", "")));
Assert.assertEquals(ImmutableList.of(0, 1, 2, 3, 4, 5), select(toInFilter("dim3", null, "a")));
Assert.assertEquals(ImmutableList.<Integer>of(), select(toInFilter("dim3", "a")));
Assert.assertEquals(ImmutableList.<Integer>of(), select(toInFilter("dim3", "b")));
Assert.assertEquals(ImmutableList.<Integer>of(), select(toInFilter("dim3", "c")));
}
private DimFilter toInFilter(String dim, String value, String... values)
{
return new InDimFilter(dim, Lists.asList(value, values));
}
private List<Integer> select(final DimFilter filter)
{
return Lists.newArrayList(
Iterables.transform(
selectColumnValuesMatchingFilter(filter, "dim0"),
new Function<String, Integer>()
{
@Override
public Integer apply(String input)
{
Preconditions.checkArgument(input.length() == 1);
return ((int) input.charAt(0)) - ((int) 'a');
}
}
)
);
}
}