From 3f048f0b155e5778fc20e707197628d8b21815c3 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Tue, 5 Jan 2016 21:36:52 -0600 Subject: [PATCH] adding support to execute Select queries in AggregationTestHelper so that Select query based UTs can be written for complex aggregator implementations --- .../theta/SketchAggregationTest.java | 2 +- .../oldapi/OldApiSketchAggregationTest.java | 2 +- .../ApproximateHistogramAggregationTest.java | 5 +- .../druid/query/MultiValuedDimensionTest.java | 2 +- .../aggregation/AggregationTestHelper.java | 129 +++++++++++++++--- .../HyperUniquesAggregationTest.java | 5 +- 6 files changed, 122 insertions(+), 23 deletions(-) diff --git a/extensions/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/SketchAggregationTest.java b/extensions/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/SketchAggregationTest.java index 35549e0e80b..f2eb1ce87ed 100644 --- a/extensions/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/SketchAggregationTest.java +++ b/extensions/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/SketchAggregationTest.java @@ -56,7 +56,7 @@ public class SketchAggregationTest { SketchModule sm = new SketchModule(); sm.configure(null); - helper = new AggregationTestHelper(sm.getJacksonModules(), tempFolder); + helper = AggregationTestHelper.createGroupByQueryAggregationTestHelper(sm.getJacksonModules(), tempFolder); } @Test diff --git a/extensions/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldApiSketchAggregationTest.java b/extensions/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldApiSketchAggregationTest.java index ad7d1937231..b7bad2c527d 100644 --- a/extensions/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldApiSketchAggregationTest.java +++ b/extensions/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldApiSketchAggregationTest.java @@ -55,7 +55,7 @@ public class OldApiSketchAggregationTest OldApiSketchModule sm = new OldApiSketchModule(); sm.configure(null); - helper = new AggregationTestHelper( + helper = AggregationTestHelper.createGroupByQueryAggregationTestHelper( sm.getJacksonModules(), tempFolder ); diff --git a/extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregationTest.java b/extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregationTest.java index 9168ca31541..27b52c18d24 100644 --- a/extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregationTest.java +++ b/extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregationTest.java @@ -43,7 +43,10 @@ public class ApproximateHistogramAggregationTest { ApproximateHistogramDruidModule module = new ApproximateHistogramDruidModule(); module.configure(null); - helper = new AggregationTestHelper(Lists.newArrayList(module.getJacksonModules()), tempFolder); + helper = AggregationTestHelper.createGroupByQueryAggregationTestHelper( + Lists.newArrayList(module.getJacksonModules()), + tempFolder + ); } @Test diff --git a/processing/src/test/java/io/druid/query/MultiValuedDimensionTest.java b/processing/src/test/java/io/druid/query/MultiValuedDimensionTest.java index 0fefbce30a6..054692025ac 100644 --- a/processing/src/test/java/io/druid/query/MultiValuedDimensionTest.java +++ b/processing/src/test/java/io/druid/query/MultiValuedDimensionTest.java @@ -72,7 +72,7 @@ public class MultiValuedDimensionTest public MultiValuedDimensionTest() throws Exception { - helper = new AggregationTestHelper( + helper = AggregationTestHelper.createGroupByQueryAggregationTestHelper( ImmutableList.of(), null ); } diff --git a/processing/src/test/java/io/druid/query/aggregation/AggregationTestHelper.java b/processing/src/test/java/io/druid/query/aggregation/AggregationTestHelper.java index e3adad4f8df..d7b39080971 100644 --- a/processing/src/test/java/io/druid/query/aggregation/AggregationTestHelper.java +++ b/processing/src/test/java/io/druid/query/aggregation/AggregationTestHelper.java @@ -19,9 +19,14 @@ package io.druid.query.aggregation; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.core.ObjectCodec; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.type.TypeFactory; import com.google.common.base.Function; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; @@ -30,6 +35,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.io.Closeables; import com.google.common.util.concurrent.ListenableFuture; +import com.metamx.common.IAE; import com.metamx.common.guava.CloseQuietly; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; @@ -46,13 +52,17 @@ import io.druid.query.FinalizeResultsQueryRunner; import io.druid.query.IntervalChunkingQueryRunnerDecorator; import io.druid.query.Query; import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryRunnerTestHelper; import io.druid.query.QueryToolChest; import io.druid.query.QueryWatcher; -import io.druid.query.groupby.GroupByQuery; import io.druid.query.groupby.GroupByQueryConfig; import io.druid.query.groupby.GroupByQueryEngine; import io.druid.query.groupby.GroupByQueryQueryToolChest; import io.druid.query.groupby.GroupByQueryRunnerFactory; +import io.druid.query.select.SelectQueryEngine; +import io.druid.query.select.SelectQueryQueryToolChest; +import io.druid.query.select.SelectQueryRunnerFactory; import io.druid.segment.IndexIO; import io.druid.segment.IndexMerger; import io.druid.segment.IndexSpec; @@ -88,17 +98,39 @@ public class AggregationTestHelper private final ObjectMapper mapper; private final IndexMerger indexMerger; private final IndexIO indexIO; - private final GroupByQueryQueryToolChest toolChest; - private final GroupByQueryRunnerFactory factory; + private final QueryToolChest toolChest; + private final QueryRunnerFactory factory; private final TemporaryFolder tempFolder; - public AggregationTestHelper(List jsonModulesToRegister, TemporaryFolder tempFoler) + private AggregationTestHelper( + ObjectMapper mapper, + IndexMerger indexMerger, + IndexIO indexIO, + QueryToolChest toolchest, + QueryRunnerFactory factory, + TemporaryFolder tempFolder, + List jsonModulesToRegister + ) { - this.tempFolder = tempFoler; - mapper = new DefaultObjectMapper(); - indexIO = TestHelper.getTestIndexIO(); - indexMerger = TestHelper.getTestIndexMerger(); + this.mapper = mapper; + this.indexMerger = indexMerger; + this.indexIO = indexIO; + this.toolChest = toolchest; + this.factory = factory; + this.tempFolder = tempFolder; + + for(Module mod : jsonModulesToRegister) { + mapper.registerModule(mod); + } + } + + public static final AggregationTestHelper createGroupByQueryAggregationTestHelper( + List jsonModulesToRegister, + TemporaryFolder tempFolder + ) + { + ObjectMapper mapper = new DefaultObjectMapper(); for(Module mod : jsonModulesToRegister) { mapper.registerModule(mod); @@ -125,17 +157,59 @@ public class AggregationTestHelper }; GroupByQueryEngine engine = new GroupByQueryEngine(configSupplier, pool); - toolChest = new GroupByQueryQueryToolChest( + GroupByQueryQueryToolChest toolchest = new GroupByQueryQueryToolChest( configSupplier, mapper, engine, pool, NoopIntervalChunkingQueryRunnerDecorator() ); - factory = new GroupByQueryRunnerFactory( + GroupByQueryRunnerFactory factory = new GroupByQueryRunnerFactory( engine, noopQueryWatcher, configSupplier, - toolChest, + toolchest, pool ); + + return new AggregationTestHelper( + mapper, + TestHelper.getTestIndexMerger(), + TestHelper.getTestIndexIO(), + toolchest, + factory, + tempFolder, + jsonModulesToRegister + ); + } + + public static final AggregationTestHelper createSelectQueryAggregationTestHelper( + List jsonModulesToRegister, + TemporaryFolder tempFolder + ) + { + ObjectMapper mapper = new DefaultObjectMapper(); + + SelectQueryQueryToolChest toolchest = new SelectQueryQueryToolChest( + new DefaultObjectMapper(), + QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator() + ); + + SelectQueryRunnerFactory factory = new SelectQueryRunnerFactory( + new SelectQueryQueryToolChest( + new DefaultObjectMapper(), + QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator() + ), + new SelectQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ); + + return new AggregationTestHelper( + mapper, + TestHelper.getTestIndexMerger(), + TestHelper.getTestIndexIO(), + toolchest, + factory, + tempFolder, + jsonModulesToRegister + ); } public Sequence createIndexAndRunQueryOnSegment( @@ -289,12 +363,12 @@ public class AggregationTestHelper //Simulates running group-by query on individual segments as historicals would do, json serialize the results //from each segment, later deserialize and merge and finally return the results - public Sequence runQueryOnSegments(final List segmentDirs, final String groupByQueryJson) throws Exception + public Sequence runQueryOnSegments(final List segmentDirs, final String queryJson) throws Exception { - return runQueryOnSegments(segmentDirs, mapper.readValue(groupByQueryJson, GroupByQuery.class)); + return runQueryOnSegments(segmentDirs, mapper.readValue(queryJson, Query.class)); } - public Sequence runQueryOnSegments(final List segmentDirs, final GroupByQuery query) + public Sequence runQueryOnSegments(final List segmentDirs, final Query query) { final List segments = Lists.transform( segmentDirs, @@ -322,7 +396,7 @@ public class AggregationTestHelper } } - public Sequence runQueryOnSegmentsObjs(final List segments, final GroupByQuery query) + public Sequence runQueryOnSegmentsObjs(final List segments, final Query query) { final FinalizeResultsQueryRunner baseRunner = new FinalizeResultsQueryRunner( toolChest.postMergeQueryDecoration( @@ -385,10 +459,11 @@ public class AggregationTestHelper ); String resultStr = mapper.writer().writeValueAsString(yielder); + TypeFactory typeFactory = mapper.getTypeFactory(); + JavaType baseType = typeFactory.constructType(toolChest.getResultTypeReference()); + List resultRows = Lists.transform( - (List)mapper.readValue( - resultStr, new TypeReference>() {} - ), + readQueryResultArrayFromString(resultStr), toolChest.makePreComputeManipulatorFn( query, MetricManipulatorFns.deserializing() @@ -402,6 +477,24 @@ public class AggregationTestHelper }; } + private List readQueryResultArrayFromString(String str) throws Exception + { + List result = new ArrayList(); + + JsonParser jp = mapper.getFactory().createParser(str); + + if (jp.nextToken() != JsonToken.START_ARRAY) { + throw new IAE("not an array [%s]", str); + } + + ObjectCodec objectCodec = jp.getCodec(); + + while(jp.nextToken() != JsonToken.END_ARRAY) { + result.add(objectCodec.readValue(jp, toolChest.getResultTypeReference())); + } + return result; + } + public static IntervalChunkingQueryRunnerDecorator NoopIntervalChunkingQueryRunnerDecorator() { return new IntervalChunkingQueryRunnerDecorator(null, null, null) { diff --git a/processing/src/test/java/io/druid/query/aggregation/hyperloglog/HyperUniquesAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/hyperloglog/HyperUniquesAggregationTest.java index d700710e3e8..3e33dcc047d 100644 --- a/processing/src/test/java/io/druid/query/aggregation/hyperloglog/HyperUniquesAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/hyperloglog/HyperUniquesAggregationTest.java @@ -41,7 +41,10 @@ public class HyperUniquesAggregationTest @Test public void testIngestAndQuery() throws Exception { - AggregationTestHelper helper = new AggregationTestHelper(Lists.newArrayList(new AggregatorsModule()), tempFolder); + AggregationTestHelper helper = AggregationTestHelper.createGroupByQueryAggregationTestHelper( + Lists.newArrayList(new AggregatorsModule()), + tempFolder + ); String metricSpec = "[{" + "\"type\": \"hyperUnique\","