diff --git a/docs/content/querying/groupbyquery.md b/docs/content/querying/groupbyquery.md index 61ff7031b02..2b90cda542e 100644 --- a/docs/content/querying/groupbyquery.md +++ b/docs/content/querying/groupbyquery.md @@ -205,4 +205,5 @@ When using the "v2" strategy, the following query context parameters apply: |`groupByStrategy`|Overrides the value of `druid.query.groupBy.defaultStrategy` for this query.| |`bufferGrouperInitialBuckets`|Overrides the value of `druid.query.groupBy.bufferGrouperInitialBuckets` for this query.| |`bufferGrouperMaxLoadFactor`|Overrides the value of `druid.query.groupBy.bufferGrouperMaxLoadFactor` for this query.| +|`maxMergingDictionarySize`|Can be used to lower the value of `druid.query.groupBy.maxMergingDictionarySize` for this query.| |`maxOnDiskStorage`|Can be used to lower the value of `druid.query.groupBy.maxOnDiskStorage` for this query.| diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQueryConfig.java b/processing/src/main/java/io/druid/query/groupby/GroupByQueryConfig.java index 4b71baa90f5..6f3c3379440 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQueryConfig.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQueryConfig.java @@ -34,6 +34,7 @@ public class GroupByQueryConfig private static final String CTX_KEY_BUFFER_GROUPER_MAX_LOAD_FACTOR = "bufferGrouperMaxLoadFactor"; private static final String CTX_KEY_BUFFER_GROUPER_MAX_SIZE = "bufferGrouperMaxSize"; private static final String CTX_KEY_MAX_ON_DISK_STORAGE = "maxOnDiskStorage"; + private static final String CTX_KEY_MAX_MERGING_DICTIONARY_SIZE = "maxMergingDictionarySize"; @JsonProperty private String defaultStrategy = GroupByStrategySelector.STRATEGY_V1; @@ -151,9 +152,13 @@ public class GroupByQueryConfig getBufferGrouperInitialBuckets() ); newConfig.maxOnDiskStorage = Math.min( - ((Number)query.getContextValue(CTX_KEY_MAX_ON_DISK_STORAGE, getMaxOnDiskStorage())).longValue(), + ((Number) query.getContextValue(CTX_KEY_MAX_ON_DISK_STORAGE, getMaxOnDiskStorage())).longValue(), getMaxOnDiskStorage() ); + newConfig.maxMergingDictionarySize = Math.min( + ((Number) query.getContextValue(CTX_KEY_MAX_MERGING_DICTIONARY_SIZE, getMaxMergingDictionarySize())).longValue(), + getMaxMergingDictionarySize() + ); return newConfig; } } diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryConfigTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryConfigTest.java new file mode 100644 index 00000000000..be080f2227c --- /dev/null +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryConfigTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.groupby; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import io.druid.granularity.QueryGranularities; +import io.druid.jackson.DefaultObjectMapper; +import org.joda.time.Interval; +import org.junit.Assert; +import org.junit.Test; + +public class GroupByQueryConfigTest +{ + private final ObjectMapper MAPPER = new DefaultObjectMapper(); + + private final ImmutableMap CONFIG_MAP = ImmutableMap + .builder() + .put("singleThreaded", "true") + .put("defaultStrategy", "v2") + .put("bufferGrouperInitialBuckets", "1") + .put("maxIntermediateRows", "2") + .put("maxResults", "3") + .put("maxOnDiskStorage", "4") + .put("maxMergingDictionarySize", "5") + .put("bufferGrouperMaxLoadFactor", "6") + .build(); + + @Test + public void testSerde() + { + final GroupByQueryConfig config = MAPPER.convertValue(CONFIG_MAP, GroupByQueryConfig.class); + + Assert.assertEquals(true, config.isSingleThreaded()); + Assert.assertEquals("v2", config.getDefaultStrategy()); + Assert.assertEquals(1, config.getBufferGrouperInitialBuckets()); + Assert.assertEquals(2, config.getMaxIntermediateRows()); + Assert.assertEquals(3, config.getMaxResults()); + Assert.assertEquals(4, config.getMaxOnDiskStorage()); + Assert.assertEquals(5, config.getMaxMergingDictionarySize()); + Assert.assertEquals(6.0, config.getBufferGrouperMaxLoadFactor(), 0.0); + } + + @Test + public void testNoOverrides() + { + final GroupByQueryConfig config = MAPPER.convertValue(CONFIG_MAP, GroupByQueryConfig.class); + final GroupByQueryConfig config2 = config.withOverrides( + GroupByQuery.builder() + .setDataSource("test") + .setInterval(new Interval("2000/P1D")) + .setGranularity(QueryGranularities.ALL) + .build() + ); + + Assert.assertEquals(true, config2.isSingleThreaded()); + Assert.assertEquals("v2", config2.getDefaultStrategy()); + Assert.assertEquals(1, config2.getBufferGrouperInitialBuckets()); + Assert.assertEquals(2, config2.getMaxIntermediateRows()); + Assert.assertEquals(3, config2.getMaxResults()); + Assert.assertEquals(4, config2.getMaxOnDiskStorage()); + Assert.assertEquals(5, config2.getMaxMergingDictionarySize()); + Assert.assertEquals(6.0, config2.getBufferGrouperMaxLoadFactor(), 0.0); + } + + @Test + public void testOverrides() + { + final GroupByQueryConfig config = MAPPER.convertValue(CONFIG_MAP, GroupByQueryConfig.class); + final GroupByQueryConfig config2 = config.withOverrides( + GroupByQuery.builder() + .setDataSource("test") + .setInterval(new Interval("2000/P1D")) + .setGranularity(QueryGranularities.ALL) + .setContext( + ImmutableMap.of( + "groupByStrategy", "v1", + "maxOnDiskStorage", 0, + "maxResults", 2, + "maxMergingDictionarySize", 3 + ) + ) + .build() + ); + + Assert.assertEquals(true, config2.isSingleThreaded()); + Assert.assertEquals("v1", config2.getDefaultStrategy()); + Assert.assertEquals(1, config2.getBufferGrouperInitialBuckets()); + Assert.assertEquals(2, config2.getMaxIntermediateRows()); + Assert.assertEquals(2, config2.getMaxResults()); + Assert.assertEquals(0, config2.getMaxOnDiskStorage()); + Assert.assertEquals(3, config2.getMaxMergingDictionarySize()); + Assert.assertEquals(6.0, config2.getBufferGrouperMaxLoadFactor(), 0.0); + } +}