From 38cb73e0cccc61f00b43ef4bb4944c0c1550787c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Mon, 20 Oct 2014 13:19:21 -0700 Subject: [PATCH 1/3] javacript tiered broker selector strategy --- ...avascriptTieredBrokerSelectorStrategy.java | 60 ++++++++++ .../router/TieredBrokerSelectorStrategy.java | 3 +- ...criptTieredBrokerSelectorStrategyTest.java | 103 ++++++++++++++++++ 3 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 server/src/main/java/io/druid/server/router/JavascriptTieredBrokerSelectorStrategy.java create mode 100644 server/src/test/java/io/druid/server/router/JavascriptTieredBrokerSelectorStrategyTest.java diff --git a/server/src/main/java/io/druid/server/router/JavascriptTieredBrokerSelectorStrategy.java b/server/src/main/java/io/druid/server/router/JavascriptTieredBrokerSelectorStrategy.java new file mode 100644 index 00000000000..cce7fb25554 --- /dev/null +++ b/server/src/main/java/io/druid/server/router/JavascriptTieredBrokerSelectorStrategy.java @@ -0,0 +1,60 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.router; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.api.client.repackaged.com.google.common.base.Throwables; +import com.google.common.base.Optional; +import io.druid.query.Query; + +import javax.script.Compilable; +import javax.script.Invocable; +import javax.script.ScriptEngine; +import javax.script.ScriptEngineManager; +import javax.script.ScriptException; + +public class JavascriptTieredBrokerSelectorStrategy implements TieredBrokerSelectorStrategy +{ + private final SelectorFunction function; + + public JavascriptTieredBrokerSelectorStrategy(@JsonProperty("function") String function) + { + final ScriptEngine engine = new ScriptEngineManager().getEngineByName("javascript"); + try { + ((Compilable)engine).compile("var apply = " + function).eval(); + } catch(ScriptException e) { + Throwables.propagate(e); + } + this.function = ((Invocable)engine).getInterface(SelectorFunction.class); + } + + @Override + public Optional getBrokerServiceName( + TieredBrokerConfig config, Query query + ) + { + return Optional.fromNullable(function.apply(config, query)); + } + + private static interface SelectorFunction + { + public String apply(TieredBrokerConfig config, Query query); + } +} diff --git a/server/src/main/java/io/druid/server/router/TieredBrokerSelectorStrategy.java b/server/src/main/java/io/druid/server/router/TieredBrokerSelectorStrategy.java index 40a7714d870..74c1091397e 100644 --- a/server/src/main/java/io/druid/server/router/TieredBrokerSelectorStrategy.java +++ b/server/src/main/java/io/druid/server/router/TieredBrokerSelectorStrategy.java @@ -29,7 +29,8 @@ import io.druid.query.Query; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes(value = { @JsonSubTypes.Type(name = "timeBoundary", value = TimeBoundaryTieredBrokerSelectorStrategy.class), - @JsonSubTypes.Type(name = "priority", value = PriorityTieredBrokerSelectorStrategy.class) + @JsonSubTypes.Type(name = "priority", value = PriorityTieredBrokerSelectorStrategy.class), + @JsonSubTypes.Type(name = "javascript", value = JavascriptTieredBrokerSelectorStrategy.class) }) public interface TieredBrokerSelectorStrategy diff --git a/server/src/test/java/io/druid/server/router/JavascriptTieredBrokerSelectorStrategyTest.java b/server/src/test/java/io/druid/server/router/JavascriptTieredBrokerSelectorStrategyTest.java new file mode 100644 index 00000000000..78a6db3c59d --- /dev/null +++ b/server/src/test/java/io/druid/server/router/JavascriptTieredBrokerSelectorStrategyTest.java @@ -0,0 +1,103 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.router; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import io.druid.query.Druids; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.aggregation.DoubleSumAggregatorFactory; +import io.druid.query.aggregation.LongSumAggregatorFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.util.LinkedHashMap; + +public class JavascriptTieredBrokerSelectorStrategyTest +{ + + @Test + public void testGetBrokerServiceName() throws Exception + { + TieredBrokerSelectorStrategy jsStrategy = new JavascriptTieredBrokerSelectorStrategy( + "function (config, query) { if (config.getTierToBrokerMap().values().size() > 0 && query.getAggregatorSpecs && query.getAggregatorSpecs().size() <= 2) { return config.getTierToBrokerMap().values().toArray()[0] } else { return config.getDefaultBrokerServiceName() } }" + ); + + final LinkedHashMap tierBrokerMap = new LinkedHashMap<>(); + tierBrokerMap.put("fast", "druid/fastBroker"); + tierBrokerMap.put("slow", "druid/broker"); + + final TieredBrokerConfig tieredBrokerConfig = new TieredBrokerConfig() + { + @Override + public String getDefaultBrokerServiceName() + { + return "druid/broker"; + } + + @Override + public LinkedHashMap getTierToBrokerMap() + { + return tierBrokerMap; + } + }; + + final Druids.TimeseriesQueryBuilder queryBuilder = Druids.newTimeseriesQueryBuilder().dataSource("test") + .intervals("2014/2015") + .aggregators( + ImmutableList.of( + new CountAggregatorFactory("count") + ) + ); + + Assert.assertEquals( + Optional.of("druid/fastBroker"), + jsStrategy.getBrokerServiceName( + tieredBrokerConfig, + queryBuilder.build() + ) + ); + + + Assert.assertEquals( + Optional.of("druid/broker"), + jsStrategy.getBrokerServiceName( + tieredBrokerConfig, + Druids.newTimeBoundaryQueryBuilder().dataSource("test").bound("maxTime").build() + ) + ); + + Assert.assertEquals( + Optional.of("druid/broker"), + jsStrategy.getBrokerServiceName( + tieredBrokerConfig, + queryBuilder.aggregators( + ImmutableList.of( + new CountAggregatorFactory("count"), + new LongSumAggregatorFactory("longSum", "a"), + new DoubleSumAggregatorFactory("doubleSum", "b") + ) + ).build() + ) + ); + + } +} From b32ce31eff5beeca3e890b461ce88dacc1e13440 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Mon, 20 Oct 2014 14:11:49 -0700 Subject: [PATCH 2/3] add documentation --- docs/content/Router.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/content/Router.md b/docs/content/Router.md index 73849164216..f046fe4b468 100644 --- a/docs/content/Router.md +++ b/docs/content/Router.md @@ -118,3 +118,16 @@ Including this strategy means all timeBoundary queries are always routed to the ``` Queries with a priority set to less than minPriority are routed to the lowest priority broker. Queries with priority set to greater than maxPriority are routed to the highest priority broker. By default, minPriority is 0 and maxPriority is 1. Using these default values, if a query with priority 0 (the default query priority is 0) is sent, the query skips the priority selection logic. + +### javascript + +Allows defining arbitrary routing rules using a JavaScript function. The function is passed the configuration and the query to be executed, and returns the tier it should be routed to, or null for the default tier. + +*Example*: a function that return the highest priority broker unless the given query has more than two aggregators. + +```json +{ + "type" : "javascript", + "function" : "function (config, query) { if (config.getTierToBrokerMap().values().size() > 0 && query.getAggregatorSpecs && query.getAggregatorSpecs().size() <= 2) { return config.getTierToBrokerMap().values().toArray()[0] } else { return config.getDefaultBrokerServiceName() } }" +} +``` From 9d5e65913da00f280c38424d70d0e7d9601129fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 21 Oct 2014 14:09:11 -0700 Subject: [PATCH 3/3] fix serde --- docs/content/Aggregations.md | 2 +- ...avascriptTieredBrokerSelectorStrategy.java | 66 ++++++++++++++++--- .../router/TieredBrokerSelectorStrategy.java | 2 +- ...criptTieredBrokerSelectorStrategyTest.java | 24 +++++-- 4 files changed, 78 insertions(+), 16 deletions(-) diff --git a/docs/content/Aggregations.md b/docs/content/Aggregations.md index 29740a2858c..abd4780b025 100644 --- a/docs/content/Aggregations.md +++ b/docs/content/Aggregations.md @@ -159,4 +159,4 @@ Uses [HyperLogLog](http://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf) to ```json { "type" : "hyperUnique", "name" : , "fieldName" : } -``` \ No newline at end of file +``` diff --git a/server/src/main/java/io/druid/server/router/JavascriptTieredBrokerSelectorStrategy.java b/server/src/main/java/io/druid/server/router/JavascriptTieredBrokerSelectorStrategy.java index cce7fb25554..629c6a6873e 100644 --- a/server/src/main/java/io/druid/server/router/JavascriptTieredBrokerSelectorStrategy.java +++ b/server/src/main/java/io/druid/server/router/JavascriptTieredBrokerSelectorStrategy.java @@ -19,9 +19,11 @@ package io.druid.server.router; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.api.client.repackaged.com.google.common.base.Throwables; import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; import io.druid.query.Query; import javax.script.Compilable; @@ -30,19 +32,29 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; -public class JavascriptTieredBrokerSelectorStrategy implements TieredBrokerSelectorStrategy +public class JavaScriptTieredBrokerSelectorStrategy implements TieredBrokerSelectorStrategy { - private final SelectorFunction function; - - public JavascriptTieredBrokerSelectorStrategy(@JsonProperty("function") String function) + private static interface SelectorFunction { + public String apply(TieredBrokerConfig config, Query query); + } + + private final SelectorFunction fnSelector; + private final String function; + + @JsonCreator + public JavaScriptTieredBrokerSelectorStrategy(@JsonProperty("function") String fn) + { + Preconditions.checkNotNull(fn, "function must not be null"); + final ScriptEngine engine = new ScriptEngineManager().getEngineByName("javascript"); try { - ((Compilable)engine).compile("var apply = " + function).eval(); + ((Compilable)engine).compile("var apply = " + fn).eval(); } catch(ScriptException e) { Throwables.propagate(e); } - this.function = ((Invocable)engine).getInterface(SelectorFunction.class); + this.function = fn; + this.fnSelector = ((Invocable)engine).getInterface(SelectorFunction.class); } @Override @@ -50,11 +62,45 @@ public class JavascriptTieredBrokerSelectorStrategy implements TieredBrokerSelec TieredBrokerConfig config, Query query ) { - return Optional.fromNullable(function.apply(config, query)); + return Optional.fromNullable(fnSelector.apply(config, query)); } - private static interface SelectorFunction + @JsonProperty + public String getFunction() { - public String apply(TieredBrokerConfig config, Query query); + return function; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + JavaScriptTieredBrokerSelectorStrategy that = (JavaScriptTieredBrokerSelectorStrategy) o; + + if (!function.equals(that.function)) { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + return function.hashCode(); + } + + @Override + public String toString() + { + return "JavascriptTieredBrokerSelectorStrategy{" + + "function='" + function + '\'' + + '}'; } } diff --git a/server/src/main/java/io/druid/server/router/TieredBrokerSelectorStrategy.java b/server/src/main/java/io/druid/server/router/TieredBrokerSelectorStrategy.java index 74c1091397e..10bffe4fb37 100644 --- a/server/src/main/java/io/druid/server/router/TieredBrokerSelectorStrategy.java +++ b/server/src/main/java/io/druid/server/router/TieredBrokerSelectorStrategy.java @@ -30,7 +30,7 @@ import io.druid.query.Query; @JsonSubTypes(value = { @JsonSubTypes.Type(name = "timeBoundary", value = TimeBoundaryTieredBrokerSelectorStrategy.class), @JsonSubTypes.Type(name = "priority", value = PriorityTieredBrokerSelectorStrategy.class), - @JsonSubTypes.Type(name = "javascript", value = JavascriptTieredBrokerSelectorStrategy.class) + @JsonSubTypes.Type(name = "javascript", value = JavaScriptTieredBrokerSelectorStrategy.class) }) public interface TieredBrokerSelectorStrategy diff --git a/server/src/test/java/io/druid/server/router/JavascriptTieredBrokerSelectorStrategyTest.java b/server/src/test/java/io/druid/server/router/JavascriptTieredBrokerSelectorStrategyTest.java index 78a6db3c59d..3f7e9f14947 100644 --- a/server/src/test/java/io/druid/server/router/JavascriptTieredBrokerSelectorStrategyTest.java +++ b/server/src/test/java/io/druid/server/router/JavascriptTieredBrokerSelectorStrategyTest.java @@ -19,8 +19,10 @@ package io.druid.server.router; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; +import io.druid.jackson.DefaultObjectMapper; import io.druid.query.Druids; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; @@ -31,15 +33,29 @@ import org.junit.Test; import java.util.LinkedHashMap; -public class JavascriptTieredBrokerSelectorStrategyTest +public class JavaScriptTieredBrokerSelectorStrategyTest { + final TieredBrokerSelectorStrategy jsStrategy = new JavaScriptTieredBrokerSelectorStrategy( + "function (config, query) { if (config.getTierToBrokerMap().values().size() > 0 && query.getAggregatorSpecs && query.getAggregatorSpecs().size() <= 2) { return config.getTierToBrokerMap().values().toArray()[0] } else { return config.getDefaultBrokerServiceName() } }" + ); + + @Test + public void testSerde() throws Exception + { + ObjectMapper mapper = new DefaultObjectMapper(); + Assert.assertEquals( + jsStrategy, + mapper.readValue( + mapper.writeValueAsString(jsStrategy), + JavaScriptTieredBrokerSelectorStrategy.class + ) + ); + } @Test public void testGetBrokerServiceName() throws Exception { - TieredBrokerSelectorStrategy jsStrategy = new JavascriptTieredBrokerSelectorStrategy( - "function (config, query) { if (config.getTierToBrokerMap().values().size() > 0 && query.getAggregatorSpecs && query.getAggregatorSpecs().size() <= 2) { return config.getTierToBrokerMap().values().toArray()[0] } else { return config.getDefaultBrokerServiceName() } }" - ); + final LinkedHashMap tierBrokerMap = new LinkedHashMap<>(); tierBrokerMap.put("fast", "druid/fastBroker");