From 4d63196535d58c238c17135612531c4a1dce200f Mon Sep 17 00:00:00 2001 From: "navis.ryu" Date: Fri, 29 Jan 2016 08:52:10 +0900 Subject: [PATCH] Support dimension spec for select query --- docs/content/querying/select-query.md | 2 +- .../src/main/java/io/druid/query/Druids.java | 14 +- .../query/dimension/DefaultDimensionSpec.java | 29 +++- .../dimension/ExtractionDimensionSpec.java | 5 + .../io/druid/query/select/PagingSpec.java | 30 ++++ .../io/druid/query/select/SelectQuery.java | 7 +- .../druid/query/select/SelectQueryEngine.java | 12 +- .../select/SelectQueryQueryToolChest.java | 14 +- .../query/select/SelectQueryRunnerTest.java | 157 ++++++++++++++++-- .../query/select/SelectQuerySpecTest.java | 82 +++++++++ 10 files changed, 317 insertions(+), 35 deletions(-) create mode 100644 processing/src/test/java/io/druid/query/select/SelectQuerySpecTest.java diff --git a/docs/content/querying/select-query.md b/docs/content/querying/select-query.md index 9bc5b8b2e06..75eaf310eaa 100644 --- a/docs/content/querying/select-query.md +++ b/docs/content/querying/select-query.md @@ -28,7 +28,7 @@ There are several main parts to a select query: |intervals|A JSON Object representing ISO-8601 Intervals. This defines the time ranges to run the query over.|yes| |descending|Whether to make descending ordered result. Default is `false`(ascending). When this is `true`, page identifier and offsets will be negative value.|no| |filter|See [Filters](../querying/filters.html)|no| -|dimensions|A String array of dimensions to select. If left empty, all dimensions are returned.|no| +|dimensions|A JSON list of dimensions to select; or see [DimensionSpec](../querying/dimensionspecs.html) for ways to extract dimensions. If left empty, all dimensions are returned.|no| |metrics|A String array of metrics to select. If left empty, all metrics are returned.|no| |pagingSpec|A JSON object indicating offsets into different scanned segments. Query results will return a `pagingIdentifiers` value that can be reused in the next query for pagination.|yes| |context|An additional JSON Object which can be used to specify certain flags.|no| diff --git a/processing/src/main/java/io/druid/query/Druids.java b/processing/src/main/java/io/druid/query/Druids.java index a971a7cda52..fe6c096160e 100644 --- a/processing/src/main/java/io/druid/query/Druids.java +++ b/processing/src/main/java/io/druid/query/Druids.java @@ -1077,7 +1077,7 @@ public class Druids private Map context; private DimFilter dimFilter; private QueryGranularity granularity; - private List dimensions; + private List dimensions; private List metrics; private PagingSpec pagingSpec; @@ -1101,9 +1101,7 @@ public class Druids descending, dimFilter, granularity, - dimensions, - metrics, - pagingSpec, + dimensions, metrics, pagingSpec, context ); } @@ -1192,12 +1190,18 @@ public class Druids return this; } - public SelectQueryBuilder dimensions(List d) + public SelectQueryBuilder dimensionSpecs(List d) { dimensions = d; return this; } + public SelectQueryBuilder dimensions(List d) + { + dimensions = DefaultDimensionSpec.toSpec(d); + return this; + } + public SelectQueryBuilder metrics(List m) { metrics = m; diff --git a/processing/src/main/java/io/druid/query/dimension/DefaultDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/DefaultDimensionSpec.java index 86ecfd38f96..4f1b6447db6 100644 --- a/processing/src/main/java/io/druid/query/dimension/DefaultDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/DefaultDimensionSpec.java @@ -21,16 +21,42 @@ package io.druid.query.dimension; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Function; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import com.metamx.common.StringUtils; import io.druid.query.extraction.ExtractionFn; import io.druid.segment.DimensionSelector; import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; /** */ public class DefaultDimensionSpec implements DimensionSpec { + public static List toSpec(String... dimensionNames) + { + return toSpec(Arrays.asList(dimensionNames)); + } + + public static List toSpec(Iterable dimensionNames) + { + return Lists.newArrayList( + Iterables.transform( + dimensionNames, new Function() + { + @Override + public DimensionSpec apply(String input) + { + return new DefaultDimensionSpec(input, input); + } + } + ) + ); + } + private static final byte CACHE_TYPE_ID = 0x0; private final String dimension; private final String outputName; @@ -103,7 +129,8 @@ public class DefaultDimensionSpec implements DimensionSpec public boolean equals(Object o) { if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + // LegacyDimensionSpec can be equal to DefaultDimensionSpec + if (!(o instanceof DefaultDimensionSpec)) return false; DefaultDimensionSpec that = (DefaultDimensionSpec) o; diff --git a/processing/src/main/java/io/druid/query/dimension/ExtractionDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/ExtractionDimensionSpec.java index ca30b89b0d4..802f3776b91 100644 --- a/processing/src/main/java/io/druid/query/dimension/ExtractionDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/ExtractionDimensionSpec.java @@ -57,6 +57,11 @@ public class ExtractionDimensionSpec implements DimensionSpec this.outputName = outputName == null ? dimension : outputName; } + public ExtractionDimensionSpec(String dimension, String outputName, ExtractionFn extractionFn) + { + this(dimension, outputName, extractionFn, null); + } + @Override @JsonProperty public String getDimension() diff --git a/processing/src/main/java/io/druid/query/select/PagingSpec.java b/processing/src/main/java/io/druid/query/select/PagingSpec.java index 86a61e8f5d8..38d5f141240 100644 --- a/processing/src/main/java/io/druid/query/select/PagingSpec.java +++ b/processing/src/main/java/io/druid/query/select/PagingSpec.java @@ -90,6 +90,36 @@ public class PagingSpec return queryCacheKey.array(); } + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (!(o instanceof PagingSpec)) { + return false; + } + + PagingSpec that = (PagingSpec) o; + + if (threshold != that.threshold) { + return false; + } + if (!pagingIdentifiers.equals(that.pagingIdentifiers)) { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + int result = pagingIdentifiers.hashCode(); + result = 31 * result + threshold; + return result; + } + @Override public String toString() { diff --git a/processing/src/main/java/io/druid/query/select/SelectQuery.java b/processing/src/main/java/io/druid/query/select/SelectQuery.java index 05603c6afd7..2e39c0f547e 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQuery.java +++ b/processing/src/main/java/io/druid/query/select/SelectQuery.java @@ -28,6 +28,7 @@ import io.druid.query.BaseQuery; import io.druid.query.DataSource; import io.druid.query.Query; import io.druid.query.Result; +import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.DimFilter; import io.druid.query.spec.QuerySegmentSpec; @@ -41,7 +42,7 @@ public class SelectQuery extends BaseQuery> { private final DimFilter dimFilter; private final QueryGranularity granularity; - private final List dimensions; + private final List dimensions; private final List metrics; private final PagingSpec pagingSpec; @@ -52,7 +53,7 @@ public class SelectQuery extends BaseQuery> @JsonProperty("descending") boolean descending, @JsonProperty("filter") DimFilter dimFilter, @JsonProperty("granularity") QueryGranularity granularity, - @JsonProperty("dimensions") List dimensions, + @JsonProperty("dimensions") List dimensions, @JsonProperty("metrics") List metrics, @JsonProperty("pagingSpec") PagingSpec pagingSpec, @JsonProperty("context") Map context @@ -104,7 +105,7 @@ public class SelectQuery extends BaseQuery> } @JsonProperty - public List getDimensions() + public List getDimensions() { return dimensions; } diff --git a/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java b/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java index 75e5ac944ad..d6f1bee5f3e 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java +++ b/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java @@ -27,6 +27,7 @@ import com.metamx.common.guava.Sequence; import io.druid.query.QueryRunnerHelper; import io.druid.query.Result; import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.dimension.DimensionSpec; import io.druid.segment.Cursor; import io.druid.segment.DimensionSelector; import io.druid.segment.LongColumnSelector; @@ -55,9 +56,9 @@ public class SelectQueryEngine ); } - final Iterable dims; + final Iterable dims; if (query.getDimensions() == null || query.getDimensions().isEmpty()) { - dims = adapter.getAvailableDimensions(); + dims = DefaultDimensionSpec.toSpec(adapter.getAvailableDimensions()); } else { dims = query.getDimensions(); } @@ -89,10 +90,9 @@ public class SelectQueryEngine final LongColumnSelector timestampColumnSelector = cursor.makeLongColumnSelector(Column.TIME_COLUMN_NAME); final Map dimSelectors = Maps.newHashMap(); - for (String dim : dims) { - // switching to using DimensionSpec for select would allow the use of extractionFn here. - final DimensionSelector dimSelector = cursor.makeDimensionSelector(new DefaultDimensionSpec(dim, dim)); - dimSelectors.put(dim, dimSelector); + for (DimensionSpec dim : dims) { + final DimensionSelector dimSelector = cursor.makeDimensionSelector(dim); + dimSelectors.put(dim.getOutputName(), dimSelector); } final Map metSelectors = Maps.newHashMap(); diff --git a/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java index 3814fa10427..eba425bd62b 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java @@ -40,11 +40,13 @@ import io.druid.query.Result; import io.druid.query.ResultGranularTimestampComparator; import io.druid.query.ResultMergeQueryRunner; import io.druid.query.aggregation.MetricManipulationFn; +import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.DimFilter; import org.joda.time.DateTime; import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -138,16 +140,16 @@ public class SelectQueryQueryToolChest extends QueryToolChest dimensions = Sets.newTreeSet(); - if (query.getDimensions() != null) { - dimensions.addAll(query.getDimensions()); + List dimensionSpecs = query.getDimensions(); + if (dimensionSpecs == null) { + dimensionSpecs = Collections.emptyList(); } - final byte[][] dimensionsBytes = new byte[dimensions.size()][]; + final byte[][] dimensionsBytes = new byte[dimensionSpecs.size()][]; int dimensionsBytesSize = 0; int index = 0; - for (String dimension : dimensions) { - dimensionsBytes[index] = StringUtils.toUtf8(dimension); + for (DimensionSpec dimension : dimensionSpecs) { + dimensionsBytes[index] = dimension.getCacheKey(); dimensionsBytesSize += dimensionsBytes[index].length; ++index; } diff --git a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java index bbf3fa2b2c5..3fc958063c8 100644 --- a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java @@ -32,6 +32,11 @@ import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerTestHelper; import io.druid.query.Result; import io.druid.query.TableDataSource; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.dimension.DimensionSpec; +import io.druid.query.dimension.ExtractionDimensionSpec; +import io.druid.query.extraction.LookupExtractionFn; +import io.druid.query.extraction.MapLookupExtractor; import io.druid.query.filter.AndDimFilter; import io.druid.query.filter.DimFilter; import io.druid.query.filter.SelectorDimFilter; @@ -130,7 +135,7 @@ public class SelectQueryRunnerTest descending, null, QueryRunnerTestHelper.allGran, - Arrays.asList(), + DefaultDimensionSpec.toSpec(Arrays.asList()), Arrays.asList(), new PagingSpec(null, 3), null @@ -150,6 +155,134 @@ public class SelectQueryRunnerTest verify(expectedResults, results); } + @Test + public void testFullOnSelectWithDimensionSpec() + { + Map map = new HashMap<>(); + map.put("automotive", "automotive0"); + map.put("business", "business0"); + map.put("entertainment", "entertainment0"); + map.put("health", "health0"); + map.put("mezzanine", "mezzanine0"); + map.put("news", "news0"); + map.put("premium", "premium0"); + map.put("technology", "technology0"); + map.put("travel", "travel0"); + + SelectQuery query = new SelectQuery( + new TableDataSource(QueryRunnerTestHelper.dataSource), + QueryRunnerTestHelper.fullOnInterval, + descending, + null, + QueryRunnerTestHelper.allGran, + Arrays.asList( + new DefaultDimensionSpec(QueryRunnerTestHelper.marketDimension, "mar"), + new ExtractionDimensionSpec( + QueryRunnerTestHelper.qualityDimension, + "qual", + new LookupExtractionFn(new MapLookupExtractor(map, true), false, null, true, false) + ), + new DefaultDimensionSpec(QueryRunnerTestHelper.placementDimension, "place") + ), Lists.newArrayList(), new PagingSpec(null, 3), + null + ); + HashMap context = new HashMap(); + Iterable> results = Sequences.toList( + runner.run(query, context), + Lists.>newArrayList() + ); + + List> expectedResultsAsc = Arrays.asList( + new Result( + new DateTime("2011-01-12T00:00:00.000Z"), + new SelectResultValue( + ImmutableMap.of(QueryRunnerTestHelper.segmentId, 2), + Arrays.asList( + new EventHolder( + QueryRunnerTestHelper.segmentId, + 0, + new ImmutableMap.Builder() + .put(EventHolder.timestampKey, new DateTime("2011-01-12T00:00:00.000Z")) + .put("mar", "spot") + .put("qual", "automotive0") + .put("place", "preferred") + .put(QueryRunnerTestHelper.indexMetric, 100.000000F) + .build() + ), + new EventHolder( + QueryRunnerTestHelper.segmentId, + 1, + new ImmutableMap.Builder() + .put(EventHolder.timestampKey, new DateTime("2011-01-12T00:00:00.000Z")) + .put("mar", "spot") + .put("qual", "business0") + .put("place", "preferred") + .put(QueryRunnerTestHelper.indexMetric, 100.000000F) + .build() + ), + new EventHolder( + QueryRunnerTestHelper.segmentId, + 2, + new ImmutableMap.Builder() + .put(EventHolder.timestampKey, new DateTime("2011-01-12T00:00:00.000Z")) + .put("mar", "spot") + .put("qual", "entertainment0") + .put("place", "preferred") + .put(QueryRunnerTestHelper.indexMetric, 100.000000F) + .build() + ) + ) + ) + ) + ); + + List> expectedResultsDsc = Arrays.asList( + new Result( + new DateTime("2011-01-12T00:00:00.000Z"), + new SelectResultValue( + ImmutableMap.of(QueryRunnerTestHelper.segmentId, -3), + Arrays.asList( + new EventHolder( + QueryRunnerTestHelper.segmentId, + -1, + new ImmutableMap.Builder() + .put(EventHolder.timestampKey, new DateTime("2011-04-15T00:00:00.000Z")) + .put("mar", "upfront") + .put("qual", "premium0") + .put("place", "preferred") + .put(QueryRunnerTestHelper.indexMetric, 780.27197265625F) + .build() + ), + new EventHolder( + QueryRunnerTestHelper.segmentId, + -2, + new ImmutableMap.Builder() + .put(EventHolder.timestampKey, new DateTime("2011-04-15T00:00:00.000Z")) + .put("mar", "upfront") + .put("qual", "mezzanine0") + .put("place", "preferred") + .put(QueryRunnerTestHelper.indexMetric, 962.731201171875F) + .build() + ), + new EventHolder( + QueryRunnerTestHelper.segmentId, + -3, + new ImmutableMap.Builder() + .put(EventHolder.timestampKey, new DateTime("2011-04-15T00:00:00.000Z")) + .put("mar", "total_market") + .put("qual", "premium0") + .put("place", "preferred") + .put(QueryRunnerTestHelper.indexMetric, 1029.0570068359375F) + .build() + ) + ) + ) + ) + ); + + verify(descending ? expectedResultsDsc : expectedResultsAsc, results); + } + @Test public void testSelectWithDimsAndMets() { @@ -159,7 +292,7 @@ public class SelectQueryRunnerTest descending, null, QueryRunnerTestHelper.allGran, - Arrays.asList(QueryRunnerTestHelper.marketDimension), + DefaultDimensionSpec.toSpec(Arrays.asList(QueryRunnerTestHelper.marketDimension)), Arrays.asList(QueryRunnerTestHelper.indexMetric), new PagingSpec(null, 3), null @@ -198,7 +331,7 @@ public class SelectQueryRunnerTest descending, null, QueryRunnerTestHelper.allGran, - Arrays.asList(QueryRunnerTestHelper.qualityDimension), + DefaultDimensionSpec.toSpec(Arrays.asList(QueryRunnerTestHelper.qualityDimension)), Arrays.asList(QueryRunnerTestHelper.indexMetric), new PagingSpec(toPagingIdentifier(3, descending), 3), null @@ -236,7 +369,7 @@ public class SelectQueryRunnerTest descending, new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "spot"), QueryRunnerTestHelper.dayGran, - Lists.newArrayList(QueryRunnerTestHelper.qualityDimension), + DefaultDimensionSpec.toSpec(Lists.newArrayList(QueryRunnerTestHelper.qualityDimension)), Lists.newArrayList(QueryRunnerTestHelper.indexMetric), new PagingSpec(toPagingIdentifier(param[0], descending), param[1]), null @@ -299,15 +432,13 @@ public class SelectQueryRunnerTest I_0112_0114, descending, new AndDimFilter( - Arrays.asList( - new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "spot"), - new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "foo") - ) - ), + Arrays.asList( + new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "spot"), + new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "foo") + ) + ), QueryRunnerTestHelper.allGran, - Lists.newArrayList(), - Lists.newArrayList(), - new PagingSpec(null, 3), + DefaultDimensionSpec.toSpec(Lists.newArrayList()), Lists.newArrayList(), new PagingSpec(null, 3), null ); @@ -338,7 +469,7 @@ public class SelectQueryRunnerTest descending, null, QueryRunnerTestHelper.allGran, - Lists.newArrayList("foo"), + DefaultDimensionSpec.toSpec(Lists.newArrayList("foo")), Lists.newArrayList("foo2"), new PagingSpec(null, 3), null diff --git a/processing/src/test/java/io/druid/query/select/SelectQuerySpecTest.java b/processing/src/test/java/io/druid/query/select/SelectQuerySpecTest.java new file mode 100644 index 00000000000..d56f51b66ee --- /dev/null +++ b/processing/src/test/java/io/druid/query/select/SelectQuerySpecTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.select; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.TableDataSource; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.spec.LegacySegmentSpec; +import org.joda.time.Interval; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; + +/** + */ +public class SelectQuerySpecTest +{ + private static final ObjectMapper jsonMapper = new DefaultObjectMapper(); + + @Test + public void testSerializationLegacyString() throws Exception + { + String legacy = + "{\"queryType\":\"select\",\"dataSource\":{\"type\":\"table\",\"name\":\"testing\"}," + + "\"intervals\":{\"type\":\"LegacySegmentSpec\",\"intervals\":[\"2011-01-12T00:00:00.000Z/2011-01-14T00:00:00.000Z\"]}," + + "\"descending\":true," + + "\"filter\":null," + + "\"granularity\":{\"type\":\"all\"}," + + "\"dimensions\":[\"market\",\"quality\"]," + + "\"metrics\":[\"index\"]," + + "\"pagingSpec\":{\"pagingIdentifiers\":{},\"threshold\":3}," + + "\"context\":null}"; + + String current = + "{\"queryType\":\"select\",\"dataSource\":{\"type\":\"table\",\"name\":\"testing\"}," + + "\"intervals\":{\"type\":\"LegacySegmentSpec\",\"intervals\":[\"2011-01-12T00:00:00.000Z/2011-01-14T00:00:00.000Z\"]}," + + "\"descending\":true," + + "\"filter\":null," + + "\"granularity\":{\"type\":\"all\"}," + + "\"dimensions\":[{\"type\":\"default\",\"dimension\":\"market\",\"outputName\":\"market\"},{\"type\":\"default\",\"dimension\":\"quality\",\"outputName\":\"quality\"}]," + + "\"metrics\":[\"index\"]," + + "\"pagingSpec\":{\"pagingIdentifiers\":{},\"threshold\":3}," + + "\"context\":null}"; + + SelectQuery query = new SelectQuery( + new TableDataSource(QueryRunnerTestHelper.dataSource), + new LegacySegmentSpec(new Interval("2011-01-12/2011-01-14")), + true, + null, + QueryRunnerTestHelper.allGran, + DefaultDimensionSpec.toSpec(Arrays.asList("market", "quality")), + Arrays.asList("index"), + new PagingSpec(null, 3), + null + ); + + String actual = jsonMapper.writeValueAsString(query); + Assert.assertEquals(current, actual); + Assert.assertEquals(query, jsonMapper.readValue(actual, SelectQuery.class)); + Assert.assertEquals(query, jsonMapper.readValue(legacy, SelectQuery.class)); + } +}