From 8a4185897eeeb63d39d5784ffe0b57c97c03f860 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Mon, 24 Jul 2017 17:02:02 -0700 Subject: [PATCH] Add filter tests for both floats and doubles. (#4597) --- .../druid/query/filter/SelectorDimFilter.java | 2 +- .../io/druid/segment/filter/BoundFilter.java | 2 +- ....java => FloatAndDoubleFilteringTest.java} | 166 ++++++++++-------- 3 files changed, 93 insertions(+), 77 deletions(-) rename processing/src/test/java/io/druid/segment/filter/{FloatFilteringTest.java => FloatAndDoubleFilteringTest.java} (65%) diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index 7cfc9fbe270..d13971a8ea2 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -125,7 +125,7 @@ public class SelectorDimFilter implements DimFilter public DruidDoublePredicate makeDoublePredicate() { initDoublePredicate(); - return input -> druidDoublePredicate.applyDouble(input); + return druidDoublePredicate; } }; return new DimensionPredicateFilter(dimension, predicateFactory, extractionFn); diff --git a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java index 73b764aa6fc..8c4f804c3c2 100644 --- a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java @@ -238,7 +238,7 @@ public class BoundFilter implements Filter return input -> doesMatch(extractionFn.apply(input)); } if (boundDimFilter.getOrdering().equals(StringComparators.NUMERIC)) { - return input -> doublePredicateSupplier.get().applyDouble(input); + return doublePredicateSupplier.get(); } return input -> doesMatch(String.valueOf(input)); } diff --git a/processing/src/test/java/io/druid/segment/filter/FloatFilteringTest.java b/processing/src/test/java/io/druid/segment/filter/FloatAndDoubleFilteringTest.java similarity index 65% rename from processing/src/test/java/io/druid/segment/filter/FloatFilteringTest.java rename to processing/src/test/java/io/druid/segment/filter/FloatAndDoubleFilteringTest.java index ab800dd597b..4ae510a720b 100644 --- a/processing/src/test/java/io/druid/segment/filter/FloatFilteringTest.java +++ b/processing/src/test/java/io/druid/segment/filter/FloatAndDoubleFilteringTest.java @@ -28,13 +28,15 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; +import io.druid.data.input.impl.DoubleDimensionSchema; +import io.druid.data.input.impl.FloatDimensionSchema; import io.druid.data.input.impl.InputRowParser; import io.druid.data.input.impl.MapInputRowParser; +import io.druid.data.input.impl.StringDimensionSchema; import io.druid.data.input.impl.TimeAndDimsParseSpec; import io.druid.data.input.impl.TimestampSpec; import io.druid.java.util.common.Pair; import io.druid.js.JavaScriptConfig; -import io.druid.query.aggregation.DoubleSumAggregatorFactory; import io.druid.query.extraction.MapLookupExtractor; import io.druid.query.filter.BoundDimFilter; import io.druid.query.filter.DimFilter; @@ -67,9 +69,10 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @RunWith(Parameterized.class) -public class FloatFilteringTest extends BaseFilterTest +public class FloatAndDoubleFilteringTest extends BaseFilterTest { private static final String FLOAT_COLUMN = "flt"; + private static final String DOUBLE_COLUMN = "dbl"; private static final String TIMESTAMP_COLUMN = "ts"; private static int EXECUTOR_NUM_THREADS = 16; private static int EXECUTOR_NUM_TASKS = 2000; @@ -78,7 +81,11 @@ public class FloatFilteringTest extends BaseFilterTest new TimeAndDimsParseSpec( new TimestampSpec(TIMESTAMP_COLUMN, "millis", new DateTime("2000")), new DimensionsSpec( - DimensionsSpec.getDefaultSchemas(ImmutableList.of("dim0", "dim1", "dim2", "dim3")), + ImmutableList.of( + new StringDimensionSchema("dim0"), + new FloatDimensionSchema("flt"), + new DoubleDimensionSchema("dbl") + ), null, null ) @@ -86,15 +93,15 @@ public class FloatFilteringTest extends BaseFilterTest ); private static final List ROWS = ImmutableList.of( - PARSER.parse(ImmutableMap.of("ts", 1L, "dim0", "1", "flt", 1.0f, "dim1", "", "dim2", ImmutableList.of("a", "b"))), - PARSER.parse(ImmutableMap.of("ts", 2L, "dim0", "2", "flt", 2.0f, "dim1", "10", "dim2", ImmutableList.of())), - PARSER.parse(ImmutableMap.of("ts", 3L, "dim0", "3", "flt", 3.0f, "dim1", "2", "dim2", ImmutableList.of(""))), - PARSER.parse(ImmutableMap.of("ts", 4L, "dim0", "4", "flt", 4.0f, "dim1", "1", "dim2", ImmutableList.of("a"))), - PARSER.parse(ImmutableMap.of("ts", 5L, "dim0", "5", "flt", 5.0f, "dim1", "def", "dim2", ImmutableList.of("c"))), - PARSER.parse(ImmutableMap.of("ts", 6L, "dim0", "6", "flt", 6.0f, "dim1", "abc")) + PARSER.parse(ImmutableMap.of("ts", 1L, "dim0", "1", "flt", 1.0f, "dbl", 1.0d)), + PARSER.parse(ImmutableMap.of("ts", 2L, "dim0", "2", "flt", 2.0f, "dbl", 2.0d)), + PARSER.parse(ImmutableMap.of("ts", 3L, "dim0", "3", "flt", 3.0f, "dbl", 3.0d)), + PARSER.parse(ImmutableMap.of("ts", 4L, "dim0", "4", "flt", 4.0f, "dbl", 4.0d)), + PARSER.parse(ImmutableMap.of("ts", 5L, "dim0", "5", "flt", 5.0f, "dbl", 5.0d)), + PARSER.parse(ImmutableMap.of("ts", 6L, "dim0", "6", "flt", 6.0f, "dbl", 6.0d)) ); - public FloatFilteringTest( + public FloatAndDoubleFilteringTest( String testName, IndexBuilder indexBuilder, Function> finisher, @@ -107,7 +114,7 @@ public class FloatFilteringTest extends BaseFilterTest ROWS, indexBuilder.schema( new IncrementalIndexSchema.Builder() - .withMetrics(new DoubleSumAggregatorFactory(FLOAT_COLUMN, FLOAT_COLUMN)) + .withDimensionsSpec(PARSER.getParseSpec().getDimensionsSpec()) .build() ), finisher, @@ -119,59 +126,76 @@ public class FloatFilteringTest extends BaseFilterTest @AfterClass public static void tearDown() throws Exception { - BaseFilterTest.tearDown(FloatFilteringTest.class.getName()); + BaseFilterTest.tearDown(FloatAndDoubleFilteringTest.class.getName()); } @Test public void testFloatColumnFiltering() { - /* assertFilterMatches( - new SelectorDimFilter(FLOAT_COLUMN, "0", null), - ImmutableList.of() - ); + doTestFloatColumnFiltering(FLOAT_COLUMN); + doTestFloatColumnFiltering(DOUBLE_COLUMN); + } - assertFilterMatches( - new SelectorDimFilter(FLOAT_COLUMN, "0.0", null), - ImmutableList.of() - );*/ + @Test + public void testFloatColumnFilteringWithNonNumbers() + { + doTestFloatColumnFilteringWithNonNumbers(FLOAT_COLUMN); + doTestFloatColumnFilteringWithNonNumbers(DOUBLE_COLUMN); + } + @Test + public void testFloatFilterWithExtractionFn() + { + doTestFloatFilterWithExtractionFn(FLOAT_COLUMN); + doTestFloatFilterWithExtractionFn(DOUBLE_COLUMN); + } + + @Test + public void testMultithreaded() + { + doTestMultithreaded(FLOAT_COLUMN); + doTestMultithreaded(DOUBLE_COLUMN); + } + + private void doTestFloatColumnFiltering(final String columnName) + { assertFilterMatches( - new SelectorDimFilter(FLOAT_COLUMN, "3", null), + new SelectorDimFilter(columnName, "3", null), ImmutableList.of("3") ); assertFilterMatches( - new SelectorDimFilter(FLOAT_COLUMN, "3.0", null), + new SelectorDimFilter(columnName, "3.0", null), ImmutableList.of("3") ); assertFilterMatches( - new BoundDimFilter(FLOAT_COLUMN, "2", "5", false, false, null, null, StringComparators.NUMERIC), + new BoundDimFilter(columnName, "2", "5", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of("2", "3", "4", "5") ); assertFilterMatches( - new BoundDimFilter(FLOAT_COLUMN, "2.0", "5.0", false, false, null, null, StringComparators.NUMERIC), + new BoundDimFilter(columnName, "2.0", "5.0", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of("2", "3", "4", "5") ); assertFilterMatches( - new BoundDimFilter(FLOAT_COLUMN, "1", "4", true, true, null, null, StringComparators.NUMERIC), + new BoundDimFilter(columnName, "1", "4", true, true, null, null, StringComparators.NUMERIC), ImmutableList.of("2", "3") ); assertFilterMatches( - new BoundDimFilter(FLOAT_COLUMN, "1.0", "4.0", true, true, null, null, StringComparators.NUMERIC), + new BoundDimFilter(columnName, "1.0", "4.0", true, true, null, null, StringComparators.NUMERIC), ImmutableList.of("2", "3") ); assertFilterMatches( - new InDimFilter(FLOAT_COLUMN, Arrays.asList("2", "4", "8"), null), + new InDimFilter(columnName, Arrays.asList("2", "4", "8"), null), ImmutableList.of("2", "4") ); assertFilterMatches( - new InDimFilter(FLOAT_COLUMN, Arrays.asList("2.0", "4.0", "8.0"), null), + new InDimFilter(columnName, Arrays.asList("2.0", "4.0", "8.0"), null), ImmutableList.of("2", "4") ); @@ -181,95 +205,93 @@ public class FloatFilteringTest extends BaseFilterTest infilterValues.add(String.valueOf(i * 2)); } assertFilterMatches( - new InDimFilter(FLOAT_COLUMN, infilterValues, null), + new InDimFilter(columnName, infilterValues, null), ImmutableList.of("2", "4", "6") ); String jsFn = "function(x) { return(x === 3 || x === 5) }"; assertFilterMatches( - new JavaScriptDimFilter(FLOAT_COLUMN, jsFn, null, JavaScriptConfig.getEnabledInstance()), + new JavaScriptDimFilter(columnName, jsFn, null, JavaScriptConfig.getEnabledInstance()), ImmutableList.of("3", "5") ); String jsFn2 = "function(x) { return(x === 3.0 || x === 5.0) }"; assertFilterMatches( - new JavaScriptDimFilter(FLOAT_COLUMN, jsFn2, null, JavaScriptConfig.getEnabledInstance()), + new JavaScriptDimFilter(columnName, jsFn2, null, JavaScriptConfig.getEnabledInstance()), ImmutableList.of("3", "5") ); assertFilterMatches( - new RegexDimFilter(FLOAT_COLUMN, "4", null), + new RegexDimFilter(columnName, "4", null), ImmutableList.of("4") ); assertFilterMatches( - new RegexDimFilter(FLOAT_COLUMN, "4.0", null), + new RegexDimFilter(columnName, "4.0", null), ImmutableList.of("4") ); assertFilterMatches( - new SearchQueryDimFilter(FLOAT_COLUMN, new ContainsSearchQuerySpec("2", true), null), + new SearchQueryDimFilter(columnName, new ContainsSearchQuerySpec("2", true), null), ImmutableList.of("2") ); assertFilterMatches( - new SearchQueryDimFilter(FLOAT_COLUMN, new ContainsSearchQuerySpec("2", true), null), + new SearchQueryDimFilter(columnName, new ContainsSearchQuerySpec("2", true), null), ImmutableList.of("2") ); } - @Test - public void testFloatColumnFilteringWithNonNumbers() + private void doTestFloatColumnFilteringWithNonNumbers(final String columnName) { assertFilterMatches( - new SelectorDimFilter(FLOAT_COLUMN, "", null), + new SelectorDimFilter(columnName, "", null), ImmutableList.of() ); assertFilterMatches( - new SelectorDimFilter(FLOAT_COLUMN, null, null), + new SelectorDimFilter(columnName, null, null), ImmutableList.of() ); assertFilterMatches( - new SelectorDimFilter(FLOAT_COLUMN, "abc", null), + new SelectorDimFilter(columnName, "abc", null), ImmutableList.of() ); assertFilterMatches( - new BoundDimFilter(FLOAT_COLUMN, "a", "b", false, false, null, null, StringComparators.NUMERIC), + new BoundDimFilter(columnName, "a", "b", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of() ); assertFilterMatches( - new BoundDimFilter(FLOAT_COLUMN, " ", "4", false, false, null, null, StringComparators.NUMERIC), + new BoundDimFilter(columnName, " ", "4", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of("1", "2", "3", "4") ); assertFilterMatches( - new BoundDimFilter(FLOAT_COLUMN, " ", "4", false, false, null, null, StringComparators.LEXICOGRAPHIC), + new BoundDimFilter(columnName, " ", "4", false, false, null, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of("1", "2", "3") ); assertFilterMatches( - new BoundDimFilter(FLOAT_COLUMN, " ", "4.0", false, false, null, null, StringComparators.LEXICOGRAPHIC), + new BoundDimFilter(columnName, " ", "4.0", false, false, null, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of("1", "2", "3", "4") ); assertFilterMatches( - new BoundDimFilter(FLOAT_COLUMN, " ", "A", false, false, null, null, StringComparators.NUMERIC), + new BoundDimFilter(columnName, " ", "A", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of() ); assertFilterMatches( - new BoundDimFilter(FLOAT_COLUMN, " ", "A", false, false, null, null, StringComparators.LEXICOGRAPHIC), + new BoundDimFilter(columnName, " ", "A", false, false, null, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of("1", "2", "3", "4", "5", "6") ); } - @Test - public void testFloatFilterWithExtractionFn() + private void doTestFloatFilterWithExtractionFn(final String columnName) { final Map stringMap = new HashMap<>(); stringMap.put("1.0", "Monday"); @@ -282,25 +304,25 @@ public class FloatFilteringTest extends BaseFilterTest LookupExtractionFn exfn = new LookupExtractionFn(mapExtractor, false, "UNKNOWN", false, true); assertFilterMatches( - new SelectorDimFilter(FLOAT_COLUMN, "Monday", exfn), + new SelectorDimFilter(columnName, "Monday", exfn), ImmutableList.of("1") ); assertFilterMatches( - new SelectorDimFilter(FLOAT_COLUMN, "Notaday", exfn), + new SelectorDimFilter(columnName, "Notaday", exfn), ImmutableList.of() ); assertFilterMatches( - new BoundDimFilter(FLOAT_COLUMN, "Fridax", "Fridaz", false, false, null, exfn, StringComparators.ALPHANUMERIC), + new BoundDimFilter(columnName, "Fridax", "Fridaz", false, false, null, exfn, StringComparators.ALPHANUMERIC), ImmutableList.of("5") ); assertFilterMatches( - new BoundDimFilter(FLOAT_COLUMN, "Friday", "Friday", true, true, null, exfn, StringComparators.ALPHANUMERIC), + new BoundDimFilter(columnName, "Friday", "Friday", true, true, null, exfn, StringComparators.ALPHANUMERIC), ImmutableList.of() ); assertFilterMatches( - new InDimFilter(FLOAT_COLUMN, Arrays.asList("Caturday", "Saturday", "Tuesday"), exfn), + new InDimFilter(columnName, Arrays.asList("Caturday", "Saturday", "Tuesday"), exfn), ImmutableList.of("2", "6") ); @@ -311,47 +333,46 @@ public class FloatFilteringTest extends BaseFilterTest "Hello", "World", "1", "2", "3", "4", "5", "6", "7" ); assertFilterMatches( - new InDimFilter(FLOAT_COLUMN, bigList, exfn), + new InDimFilter(columnName, bigList, exfn), ImmutableList.of("2", "6") ); String jsFn = "function(x) { return(x === 'Wednesday' || x === 'Thursday') }"; assertFilterMatches( - new JavaScriptDimFilter(FLOAT_COLUMN, jsFn, exfn, JavaScriptConfig.getEnabledInstance()), + new JavaScriptDimFilter(columnName, jsFn, exfn, JavaScriptConfig.getEnabledInstance()), ImmutableList.of("3", "4") ); assertFilterMatches( - new RegexDimFilter(FLOAT_COLUMN, ".*day", exfn), + new RegexDimFilter(columnName, ".*day", exfn), ImmutableList.of("1", "2", "3", "4", "5", "6") ); assertFilterMatches( - new SearchQueryDimFilter(FLOAT_COLUMN, new ContainsSearchQuerySpec("s", true), exfn), + new SearchQueryDimFilter(columnName, new ContainsSearchQuerySpec("s", true), exfn), ImmutableList.of("2", "3", "4") ); } - @Test - public void testMultithreaded() + private void doTestMultithreaded(final String columnName) { assertFilterMatchesMultithreaded( - new SelectorDimFilter(FLOAT_COLUMN, "3", null), + new SelectorDimFilter(columnName, "3", null), ImmutableList.of("3") ); assertFilterMatchesMultithreaded( - new SelectorDimFilter(FLOAT_COLUMN, "3.0", null), + new SelectorDimFilter(columnName, "3.0", null), ImmutableList.of("3") ); assertFilterMatchesMultithreaded( - new InDimFilter(FLOAT_COLUMN, Arrays.asList("2", "4", "8"), null), + new InDimFilter(columnName, Arrays.asList("2", "4", "8"), null), ImmutableList.of("2", "4") ); assertFilterMatchesMultithreaded( - new InDimFilter(FLOAT_COLUMN, Arrays.asList("2.0", "4.0", "8.0"), null), + new InDimFilter(columnName, Arrays.asList("2.0", "4.0", "8.0"), null), ImmutableList.of("2", "4") ); @@ -361,17 +382,17 @@ public class FloatFilteringTest extends BaseFilterTest infilterValues.add(String.valueOf(i * 2)); } assertFilterMatchesMultithreaded( - new InDimFilter(FLOAT_COLUMN, infilterValues, null), + new InDimFilter(columnName, infilterValues, null), ImmutableList.of("2", "4", "6") ); assertFilterMatches( - new BoundDimFilter(FLOAT_COLUMN, "2", "5", false, false, null, null, StringComparators.NUMERIC), + new BoundDimFilter(columnName, "2", "5", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of("2", "3", "4", "5") ); assertFilterMatches( - new BoundDimFilter(FLOAT_COLUMN, "2.0", "5.0", false, false, null, null, StringComparators.NUMERIC), + new BoundDimFilter(columnName, "2.0", "5.0", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of("2", "3", "4", "5") ); } @@ -389,14 +410,7 @@ public class FloatFilteringTest extends BaseFilterTest final List expectedRows ) { - return new Runnable() - { - @Override - public void run() - { - assertFilterMatches(filter, expectedRows); - } - }; + return () -> assertFilterMatches(filter, expectedRows); } private void testWithExecutor( @@ -404,7 +418,9 @@ public class FloatFilteringTest extends BaseFilterTest final List expectedRows ) { - ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(EXECUTOR_NUM_THREADS)); + ListeningExecutorService executor = MoreExecutors.listeningDecorator( + Executors.newFixedThreadPool(EXECUTOR_NUM_THREADS) + ); List> futures = new ArrayList<>();