From 074589ffcea63b5a9ac3019050bdece2338870a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Thu, 7 Nov 2013 14:10:41 -0800 Subject: [PATCH] JavaScript post-aggregator --- .../io/druid/jackson/AggregatorsModule.java | 4 +- .../post/JavaScriptPostAggregator.java | 145 ++++++++++++++++++ .../post/JavaScriptPostAggregatorTest.java | 46 ++++++ 3 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 processing/src/main/java/io/druid/query/aggregation/post/JavaScriptPostAggregator.java create mode 100644 processing/src/test/java/io/druid/query/aggregation/post/JavaScriptPostAggregatorTest.java diff --git a/processing/src/main/java/io/druid/jackson/AggregatorsModule.java b/processing/src/main/java/io/druid/jackson/AggregatorsModule.java index 8ab41053c30..15a76639997 100644 --- a/processing/src/main/java/io/druid/jackson/AggregatorsModule.java +++ b/processing/src/main/java/io/druid/jackson/AggregatorsModule.java @@ -34,6 +34,7 @@ import io.druid.query.aggregation.PostAggregator; import io.druid.query.aggregation.post.ArithmeticPostAggregator; import io.druid.query.aggregation.post.ConstantPostAggregator; import io.druid.query.aggregation.post.FieldAccessPostAggregator; +import io.druid.query.aggregation.post.JavaScriptPostAggregator; /** */ @@ -63,7 +64,8 @@ public class AggregatorsModule extends SimpleModule @JsonSubTypes(value = { @JsonSubTypes.Type(name = "arithmetic", value = ArithmeticPostAggregator.class), @JsonSubTypes.Type(name = "fieldAccess", value = FieldAccessPostAggregator.class), - @JsonSubTypes.Type(name = "constant", value = ConstantPostAggregator.class) + @JsonSubTypes.Type(name = "constant", value = ConstantPostAggregator.class), + @JsonSubTypes.Type(name = "javascript", value = JavaScriptPostAggregator.class) }) public static interface PostAggregatorMixin {} } diff --git a/processing/src/main/java/io/druid/query/aggregation/post/JavaScriptPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/JavaScriptPostAggregator.java new file mode 100644 index 00000000000..fea0c82728d --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/post/JavaScriptPostAggregator.java @@ -0,0 +1,145 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.aggregation.post; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; +import io.druid.query.aggregation.PostAggregator; +import org.mozilla.javascript.Context; +import org.mozilla.javascript.ContextFactory; +import org.mozilla.javascript.ScriptableObject; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; + + +public class JavaScriptPostAggregator implements PostAggregator +{ + private static final Comparator COMPARATOR = new Comparator() + { + @Override + public int compare(Object o, Object o1) + { + return ((Double) o).compareTo((Double) o1); + } + }; + + private static interface Function + { + public double apply(final Object[] args); + } + + private static Function compile(String function) { + final ContextFactory contextFactory = ContextFactory.getGlobal(); + Context context = contextFactory.enterContext(); + context.setOptimizationLevel(9); + + final ScriptableObject scope = context.initStandardObjects(); + + final org.mozilla.javascript.Function fn = context.compileFunction(scope, function, "aggregate", 1, null); + Context.exit(); + + + return new Function() + { + public double apply(Object[] args) + { + // ideally we need a close() function to discard the context once it is not used anymore + Context cx = Context.getCurrentContext(); + if (cx == null) { + cx = contextFactory.enterContext(); + } + + return Context.toNumber(fn.call(cx, scope, scope, args)); + } + }; + } + + private final String name; + private final List fieldNames; + private final String function; + + private final Function fn; + + + @JsonCreator + public JavaScriptPostAggregator( + @JsonProperty("name") String name, + @JsonProperty("fieldNames") final List fieldNames, + @JsonProperty("function") final String function + ) + { + Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); + Preconditions.checkNotNull(fieldNames, "Must have a valid, non-null fieldNames"); + Preconditions.checkNotNull(function, "Must have a valid, non-null fnAggregate"); + + this.name = name; + this.fieldNames = fieldNames; + this.function = function; + + this.fn = compile(function); + } + + @Override + public Set getDependentFields() + { + return Sets.newHashSet(fieldNames); + } + + @Override + public Comparator getComparator() + { + return COMPARATOR; + } + + @Override + public Object compute(Map combinedAggregators) + { + Object[] args = new Object[fieldNames.size()]; + int i = 0; + for(String field : fieldNames) { + args[i++] = combinedAggregators.get(field); + } + return fn.apply(args); + } + + @JsonProperty + @Override + public String getName() + { + return name; + } + + @JsonProperty + public List getFieldNames() + { + return fieldNames; + } + + @JsonProperty + public String getFunction() + { + return function; + } +} diff --git a/processing/src/test/java/io/druid/query/aggregation/post/JavaScriptPostAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/post/JavaScriptPostAggregatorTest.java new file mode 100644 index 00000000000..e863e7ec3db --- /dev/null +++ b/processing/src/test/java/io/druid/query/aggregation/post/JavaScriptPostAggregatorTest.java @@ -0,0 +1,46 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.aggregation.post; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Map; + +public class JavaScriptPostAggregatorTest +{ + @Test + public void testCompute() + { + JavaScriptPostAggregator javaScriptPostAggregator; + + Map metricValues = Maps.newHashMap(); + metricValues.put("delta", -10.0); + metricValues.put("total", 100.0); + + + String absPercentFunction = "function(delta, total) { return 100 * Math.abs(delta) / total; }"; + javaScriptPostAggregator = new JavaScriptPostAggregator("absPercent", Lists.newArrayList("delta", "total"), absPercentFunction); + + Assert.assertEquals(10.0, javaScriptPostAggregator.compute(metricValues)); + } +}