From 4c33d0a00ffb454811bdac6923206b2211469f14 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 28 Jun 2017 10:15:58 -0700 Subject: [PATCH] Add some new expression functions and macros. (#4442) * Add some new expression functions and macros. See misc/math-expr.md for the list of added functions, except for "like", which previously existed but was not documented. * Add easymock to datasketches tests. * Add easymock to distinctcount tests. * Add easymock to virtual-columns tests. * Code review comments. * Clean up code a bit. * Add easymock to scan-query tests. * Rework ExprMacros that have multiple impls. * Improve test coverage. --- .../java/io/druid/math/expr/Function.java | 264 +++++++++++++++++- .../java/io/druid/math/expr/FunctionTest.java | 111 ++++++++ docs/content/misc/math-expr.md | 35 ++- extensions-contrib/distinctcount/pom.xml | 5 + extensions-contrib/scan-query/pom.xml | 5 + extensions-contrib/virtual-columns/pom.xml | 5 + extensions-core/datasketches/pom.xml | 5 + .../io/druid/query/expression/ExprUtils.java | 43 +++ .../druid/query/expression/LikeExprMacro.java | 3 +- .../query/expression/LookupExprMacro.java | 93 ++++++ .../expression/RegexpExtractExprMacro.java | 80 ++++++ .../expression/TimestampCeilExprMacro.java | 117 ++++++++ .../expression/TimestampExtractExprMacro.java | 129 +++++++++ .../expression/TimestampFloorExprMacro.java | 117 ++++++++ .../expression/TimestampFormatExprMacro.java | 90 ++++++ .../expression/TimestampParseExprMacro.java | 89 ++++++ .../expression/TimestampShiftExprMacro.java | 133 +++++++++ .../druid/query/expression/ExprMacroTest.java | 143 ++++++++++ .../query/expression/TestExprMacroTable.java | 68 ++++- .../java/io/druid/guice/ExpressionModule.java | 14 + .../io/druid/query/lookup/LookupModule.java | 3 + 21 files changed, 1544 insertions(+), 8 deletions(-) create mode 100644 common/src/test/java/io/druid/math/expr/FunctionTest.java create mode 100644 processing/src/main/java/io/druid/query/expression/LookupExprMacro.java create mode 100644 processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java create mode 100644 processing/src/main/java/io/druid/query/expression/TimestampCeilExprMacro.java create mode 100644 processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java create mode 100644 processing/src/main/java/io/druid/query/expression/TimestampFloorExprMacro.java create mode 100644 processing/src/main/java/io/druid/query/expression/TimestampFormatExprMacro.java create mode 100644 processing/src/main/java/io/druid/query/expression/TimestampParseExprMacro.java create mode 100644 processing/src/main/java/io/druid/query/expression/TimestampShiftExprMacro.java create mode 100644 processing/src/test/java/io/druid/query/expression/ExprMacroTest.java diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java index a27e3c55fb1..52739d42f3c 100644 --- a/common/src/main/java/io/druid/math/expr/Function.java +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -19,6 +19,7 @@ package io.druid.math.expr; +import com.google.common.base.Strings; import io.druid.java.util.common.IAE; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; @@ -41,7 +42,7 @@ interface Function public ExprEval apply(List args, Expr.ObjectBinding bindings) { if (args.size() != 1) { - throw new IAE("function '%s' needs 1 argument", name()); + throw new IAE("Function[%s] needs 1 argument", name()); } Expr expr = args.get(0); return eval(expr.eval(bindings)); @@ -56,7 +57,7 @@ interface Function public ExprEval apply(List args, Expr.ObjectBinding bindings) { if (args.size() != 2) { - throw new IAE("function '%s' needs 2 arguments", name()); + throw new IAE("Function[%s] needs 2 arguments", name()); } Expr expr1 = args.get(0); Expr expr2 = args.get(1); @@ -242,6 +243,27 @@ interface Function } } + class Div extends DoubleParamMath + { + @Override + public String name() + { + return "div"; + } + + @Override + protected ExprEval eval(final long x, final long y) + { + return ExprEval.of(x / y); + } + + @Override + protected ExprEval eval(final double x, final double y) + { + return ExprEval.of((long) (x / y)); + } + } + class Exp extends SingleParamMath { @Override @@ -686,7 +708,7 @@ interface Function public ExprEval apply(List args, Expr.ObjectBinding bindings) { if (args.size() != 3) { - throw new IAE("function 'if' needs 3 arguments"); + throw new IAE("Function[%s] needs 3 arguments", name()); } ExprEval x = args.get(0).eval(bindings); @@ -694,6 +716,70 @@ interface Function } } + /** + * "Searched CASE" function, similar to {@code CASE WHEN boolean_expr THEN result [ELSE else_result] END} in SQL. + */ + class CaseSearchedFunc implements Function + { + @Override + public String name() + { + return "case_searched"; + } + + @Override + public ExprEval apply(final List args, final Expr.ObjectBinding bindings) + { + if (args.size() < 2) { + throw new IAE("Function[%s] must have at least 2 arguments", name()); + } + + for (int i = 0; i < args.size(); i += 2) { + if (i == args.size() - 1) { + // ELSE else_result. + return args.get(i).eval(bindings); + } else if (args.get(i).eval(bindings).asBoolean()) { + // Matching WHEN boolean_expr THEN result + return args.get(i + 1).eval(bindings); + } + } + + return ExprEval.of(null); + } + } + + /** + * "Simple CASE" function, similar to {@code CASE expr WHEN value THEN result [ELSE else_result] END} in SQL. + */ + class CaseSimpleFunc implements Function + { + @Override + public String name() + { + return "case_simple"; + } + + @Override + public ExprEval apply(final List args, final Expr.ObjectBinding bindings) + { + if (args.size() < 3) { + throw new IAE("Function[%s] must have at least 3 arguments", name()); + } + + for (int i = 1; i < args.size(); i += 2) { + if (i == args.size() - 1) { + // ELSE else_result. + return args.get(i).eval(bindings); + } else if (new BinEqExpr("==", args.get(0), args.get(i)).eval(bindings).asBoolean()) { + // Matching WHEN value THEN result + return args.get(i + 1).eval(bindings); + } + } + + return ExprEval.of(null); + } + } + class CastFunc extends DoubleParam { @Override @@ -728,7 +814,7 @@ interface Function public ExprEval apply(List args, Expr.ObjectBinding bindings) { if (args.size() != 1 && args.size() != 2) { - throw new IAE("function '%s' needs 1 or 2 arguments", name()); + throw new IAE("Function[%s] needs 1 or 2 arguments", name()); } ExprEval value = args.get(0).eval(bindings); if (value.type() != ExprType.STRING) { @@ -786,10 +872,178 @@ interface Function public ExprEval apply(List args, Expr.ObjectBinding bindings) { if (args.size() != 2) { - throw new IAE("function 'nvl' needs 2 arguments"); + throw new IAE("Function[%s] needs 2 arguments", name()); } final ExprEval eval = args.get(0).eval(bindings); return eval.isNull() ? args.get(1).eval(bindings) : eval; } } + + class ConcatFunc implements Function + { + @Override + public String name() + { + return "concat"; + } + + @Override + public ExprEval apply(List args, Expr.ObjectBinding bindings) + { + if (args.size() == 0) { + return ExprEval.of(null); + } else { + // Pass first argument in to the constructor to provide StringBuilder a little extra sizing hint. + final StringBuilder builder = new StringBuilder(Strings.nullToEmpty(args.get(0).eval(bindings).asString())); + for (int i = 1; i < args.size(); i++) { + final String s = args.get(i).eval(bindings).asString(); + if (s != null) { + builder.append(s); + } + } + return ExprEval.of(builder.toString()); + } + } + } + + class StrlenFunc implements Function + { + @Override + public String name() + { + return "strlen"; + } + + @Override + public ExprEval apply(List args, Expr.ObjectBinding bindings) + { + if (args.size() != 1) { + throw new IAE("Function[%s] needs 1 argument", name()); + } + + final String arg = args.get(0).eval(bindings).asString(); + return arg == null ? ExprEval.of(0) : ExprEval.of(arg.length()); + } + } + + class SubstringFunc implements Function + { + @Override + public String name() + { + return "substring"; + } + + @Override + public ExprEval apply(List args, Expr.ObjectBinding bindings) + { + if (args.size() != 3) { + throw new IAE("Function[%s] needs 3 arguments", name()); + } + + final String arg = args.get(0).eval(bindings).asString(); + + if (arg == null) { + return ExprEval.of(null); + } + + // Behaves like SubstringDimExtractionFn, not SQL SUBSTRING + final int index = args.get(1).eval(bindings).asInt(); + final int length = args.get(2).eval(bindings).asInt(); + + if (index < arg.length()) { + if (length >= 0) { + return ExprEval.of(arg.substring(index, Math.min(index + length, arg.length()))); + } else { + return ExprEval.of(arg.substring(index)); + } + } else { + return ExprEval.of(null); + } + } + } + + class ReplaceFunc implements Function + { + @Override + public String name() + { + return "replace"; + } + + @Override + public ExprEval apply(List args, Expr.ObjectBinding bindings) + { + if (args.size() != 3) { + throw new IAE("Function[%s] needs 3 arguments", name()); + } + + final String arg = args.get(0).eval(bindings).asString(); + final String pattern = args.get(1).eval(bindings).asString(); + final String replacement = args.get(2).eval(bindings).asString(); + return ExprEval.of( + Strings.nullToEmpty(arg).replace(Strings.nullToEmpty(pattern), Strings.nullToEmpty(replacement)) + ); + } + } + + class TrimFunc implements Function + { + @Override + public String name() + { + return "trim"; + } + + @Override + public ExprEval apply(List args, Expr.ObjectBinding bindings) + { + if (args.size() != 1) { + throw new IAE("Function[%s] needs 1 argument", name()); + } + + final String arg = args.get(0).eval(bindings).asString(); + return ExprEval.of(Strings.nullToEmpty(arg).trim()); + } + } + + class LowerFunc implements Function + { + @Override + public String name() + { + return "lower"; + } + + @Override + public ExprEval apply(List args, Expr.ObjectBinding bindings) + { + if (args.size() != 1) { + throw new IAE("Function[%s] needs 1 argument", name()); + } + + final String arg = args.get(0).eval(bindings).asString(); + return ExprEval.of(Strings.nullToEmpty(arg).toLowerCase()); + } + } + + class UpperFunc implements Function + { + @Override + public String name() + { + return "upper"; + } + + @Override + public ExprEval apply(List args, Expr.ObjectBinding bindings) + { + if (args.size() != 1) { + throw new IAE("Function[%s] needs 1 argument", name()); + } + + final String arg = args.get(0).eval(bindings).asString(); + return ExprEval.of(Strings.nullToEmpty(arg).toUpperCase()); + } + } } diff --git a/common/src/test/java/io/druid/math/expr/FunctionTest.java b/common/src/test/java/io/druid/math/expr/FunctionTest.java new file mode 100644 index 00000000000..0f731d6f7e5 --- /dev/null +++ b/common/src/test/java/io/druid/math/expr/FunctionTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +import com.google.common.collect.ImmutableMap; +import org.junit.Assert; +import org.junit.Test; + +public class FunctionTest +{ + private final Expr.ObjectBinding bindings = Parser.withMap( + ImmutableMap.of( + "x", "foo", + "y", 2, + "z", 3.1 + ) + ); + + @Test + public void testCaseSimple() + { + assertExpr("case_simple(x,'baz','is baz','foo','is foo','is other')", "is foo"); + assertExpr("case_simple(x,'baz','is baz','bar','is bar','is other')", "is other"); + assertExpr("case_simple(y,2,'is 2',3,'is 3','is other')", "is 2"); + assertExpr("case_simple(z,2,'is 2',3,'is 3','is other')", "is other"); + } + + @Test + public void testCaseSearched() + { + assertExpr("case_searched(x=='baz','is baz',x=='foo','is foo','is other')", "is foo"); + assertExpr("case_searched(x=='baz','is baz',x=='bar','is bar','is other')", "is other"); + assertExpr("case_searched(y==2,'is 2',y==3,'is 3','is other')", "is 2"); + assertExpr("case_searched(z==2,'is 2',z==3,'is 3','is other')", "is other"); + } + + @Test + public void testConcat() + { + assertExpr("concat(x,' ',y)", "foo 2"); + assertExpr("concat(x,' ',nonexistent,' ',y)", "foo 2"); + assertExpr("concat(z)", "3.1"); + assertExpr("concat()", null); + } + + @Test + public void testReplace() + { + assertExpr("replace(x,'oo','ab')", "fab"); + assertExpr("replace(x,x,'ab')", "ab"); + assertExpr("replace(x,'oo',y)", "f2"); + } + + @Test + public void testSubstring() + { + assertExpr("substring(x,0,2)", "fo"); + assertExpr("substring(x,1,2)", "oo"); + assertExpr("substring(x,y,1)", "o"); + assertExpr("substring(x,0,-1)", "foo"); + assertExpr("substring(x,0,100)", "foo"); + } + + @Test + public void testStrlen() + { + assertExpr("strlen(x)", 3L); + assertExpr("strlen(nonexistent)", 0L); + } + + @Test + public void testTrim() + { + assertExpr("trim(concat(' ',x,' '))", "foo"); + } + + @Test + public void testLower() + { + assertExpr("lower('FOO')", "foo"); + } + + @Test + public void testUpper() + { + assertExpr("upper(x)", "FOO"); + } + + private void assertExpr(final String expression, final Object expectedResult) + { + final Expr expr = Parser.parse(expression, ExprMacroTable.nil()); + Assert.assertEquals(expression, expectedResult, expr.eval(bindings).value()); + } +} diff --git a/docs/content/misc/math-expr.md b/docs/content/misc/math-expr.md index e44b19f9e14..fb40fcf1a1c 100644 --- a/docs/content/misc/math-expr.md +++ b/docs/content/misc/math-expr.md @@ -21,16 +21,48 @@ For logical operators, a number is true if and only if it is positive (0 or minu Also, the following built-in functions are supported. +## General functions + |name|description| |----|-----------| |cast|cast(expr,'LONG' or 'DOUBLE' or 'STRING') returns expr with specified type. exception can be thrown | |if|if(predicate,then,else) returns 'then' if 'predicate' evaluates to a positive number, otherwise it returns 'else' | |nvl|nvl(expr,expr-for-null) returns 'expr-for-null' if 'expr' is null (or empty string for string type) | |like|like(expr, pattern[, escape]) is equivalent to SQL `expr LIKE pattern`| +|case_searched|case_searched(expr1, result1, \[\[expr2, result2, ...\], else-result\])| +|case_simple|case_simple(expr, value1, result1, \[\[value2, result2, ...\], else-result\])| + +## String functions + +|name|description| +|----|-----------| +|concat|concatenate a list of strings| +|like|like(expr, pattern[, escape]) is equivalent to SQL `expr LIKE pattern`| +|lookup|lookup(expr, lookup-name) looks up expr in a registered [query-time lookup](lookups.html)| +|regexp_extract|regexp_extract(expr, pattern[, index]) applies a regular expression pattern and extracts a capture group index, or null if there is no match. If index is unspecified or zero, returns the substring that matched the pattern.| +|replace|replace(expr, pattern, replacement) replaces pattern with replacement| +|substring|substring(expr, index, length) behaves like java.lang.String's substring| +|strlen|returns length of a string in UTF-16 code units| +|trim|remove leading and trailing whitespace from a string| +|lower|convert a string to lowercase| +|upper|convert a string to uppercase| + +## Time functions + +|name|description| +|----|-----------| |timestamp|timestamp(expr[,format-string]) parses string expr into date then returns milli-seconds from java epoch. without 'format-string' it's regarded as ISO datetime format | |unix_timestamp|same with 'timestamp' function but returns seconds instead | +|timestamp_ceil|timestamp_ceil(expr, period, \[origin, \[timezone\]\]) rounds up a timestamp, returning it as a new timestamp. Period can be any ISO8601 period, like P3M (quarters) or PT12H (half-days). The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00".| +|timestamp_floor|timestamp_floor(expr, period, \[origin, [timezone\]\]) rounds down a timestamp, returning it as a new timestamp. Period can be any ISO8601 period, like P3M (quarters) or PT12H (half-days). The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00".| +|timestamp_shift|timestamp_shift(expr, period, step, \[timezone\]) shifts a timestamp by a period (step times), returning it as a new timestamp. Period can be any ISO8601 period. Step may be negative. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00".| +|timestamp_extract|timestamp_extract(expr, unit, \[timezone\]) extracts a time part from expr, returning it as a number. Unit can be EPOCH, SECOND, MINUTE, HOUR, DAY (day of month), DOW (day of week), DOY (day of year), WEEK (week of [week year](https://en.wikipedia.org/wiki/ISO_week_date)), MONTH (1 through 12), QUARTER (1 through 4), or YEAR. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00"| +|timestamp_parse|timestamp_parse(string expr, \[pattern, [timezone\]\]) parses a string into a timestamp using a given [Joda DateTimeFormat pattern](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html), or ISO8601 if the pattern is not provided. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00", and will be used as the time zone for strings that do not include a time zone offset. Pattern and time zone must be literals. Strings that cannot be parsed as timestamps will be returned as nulls.| +|timestamp_format|timestamp_format(expr, \[pattern, \[timezone\]\]) formats a timestamp as a string with a given [Joda DateTimeFormat pattern](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html), or ISO8601 if the pattern is not provided. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00". Pattern and time zone must be literals.| -And built-in math functions. See javadoc of java.lang.Math for detailed explanation for each function. +## Math functions + +See javadoc of java.lang.Math for detailed explanation for each function. |name|description| |----|-----------| @@ -44,6 +76,7 @@ And built-in math functions. See javadoc of java.lang.Math for detailed explanat |copysign|copysign(x) would return the first floating-point argument with the sign of the second floating-point argument| |cos|cos(x) would return the trigonometric cosine of x| |cosh|cosh(x) would return the hyperbolic cosine of x| +|div|div(x,y) is integer division of x by y| |exp|exp(x) would return Euler's number raised to the power of x| |expm1|expm1(x) would return e^x-1| |floor|floor(x) would return the largest (closest to positive infinity) double value that is less than or equal to x and is equal to a mathematical integer| diff --git a/extensions-contrib/distinctcount/pom.xml b/extensions-contrib/distinctcount/pom.xml index 33617433392..b680a6c23ed 100644 --- a/extensions-contrib/distinctcount/pom.xml +++ b/extensions-contrib/distinctcount/pom.xml @@ -55,6 +55,11 @@ junit test + + org.easymock + easymock + test + diff --git a/extensions-contrib/scan-query/pom.xml b/extensions-contrib/scan-query/pom.xml index 0d3b78d462d..3972723a849 100644 --- a/extensions-contrib/scan-query/pom.xml +++ b/extensions-contrib/scan-query/pom.xml @@ -46,6 +46,11 @@ junit test + + org.easymock + easymock + test + io.druid druid-processing diff --git a/extensions-contrib/virtual-columns/pom.xml b/extensions-contrib/virtual-columns/pom.xml index c2c158e0b4b..b6e57f41c73 100644 --- a/extensions-contrib/virtual-columns/pom.xml +++ b/extensions-contrib/virtual-columns/pom.xml @@ -59,6 +59,11 @@ junit test + + org.easymock + easymock + test + io.druid druid-processing diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index eaca6d3e5e9..000ad6e76b0 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -108,6 +108,11 @@ junit test + + org.easymock + easymock + test + io.druid druid-processing diff --git a/processing/src/main/java/io/druid/query/expression/ExprUtils.java b/processing/src/main/java/io/druid/query/expression/ExprUtils.java index 627e8ebe912..c215de9db41 100644 --- a/processing/src/main/java/io/druid/query/expression/ExprUtils.java +++ b/processing/src/main/java/io/druid/query/expression/ExprUtils.java @@ -19,7 +19,12 @@ package io.druid.query.expression; +import io.druid.java.util.common.IAE; +import io.druid.java.util.common.granularity.PeriodGranularity; import io.druid.math.expr.Expr; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.Period; public class ExprUtils { @@ -29,4 +34,42 @@ public class ExprUtils { return NIL_BINDINGS; } + + public static DateTimeZone toTimeZone(final Expr timeZoneArg) + { + if (!timeZoneArg.isLiteral()) { + throw new IAE("Time zone must be a literal"); + } + + final Object literalValue = timeZoneArg.getLiteralValue(); + return literalValue == null ? DateTimeZone.UTC : DateTimeZone.forID((String) literalValue); + } + + public static PeriodGranularity toPeriodGranularity( + final Expr periodArg, + final Expr originArg, + final Expr timeZoneArg, + final Expr.ObjectBinding bindings + ) + { + final Period period = new Period(periodArg.eval(bindings).asString()); + final DateTime origin; + final DateTimeZone timeZone; + + if (originArg == null) { + origin = null; + } else { + final Object value = originArg.eval(bindings).value(); + origin = value != null ? new DateTime(value) : null; + } + + if (timeZoneArg == null) { + timeZone = null; + } else { + final String value = timeZoneArg.eval(bindings).asString(); + timeZone = value != null ? DateTimeZone.forID(value) : null; + } + + return new PeriodGranularity(period, origin, timeZone); + } } diff --git a/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java b/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java index c2d973c3f12..c60400be33e 100644 --- a/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java @@ -19,6 +19,7 @@ package io.druid.query.expression; +import com.google.common.base.Strings; import io.druid.java.util.common.IAE; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; @@ -62,7 +63,7 @@ public class LikeExprMacro implements ExprMacroTable.ExprMacro } final LikeDimFilter.LikeMatcher likeMatcher = LikeDimFilter.LikeMatcher.from( - (String) patternExpr.getLiteralValue(), + Strings.nullToEmpty((String) patternExpr.getLiteralValue()), escapeChar ); diff --git a/processing/src/main/java/io/druid/query/expression/LookupExprMacro.java b/processing/src/main/java/io/druid/query/expression/LookupExprMacro.java new file mode 100644 index 00000000000..77dc88c184b --- /dev/null +++ b/processing/src/main/java/io/druid/query/expression/LookupExprMacro.java @@ -0,0 +1,93 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.expression; + +import com.google.common.base.Strings; +import com.google.inject.Inject; +import io.druid.java.util.common.IAE; +import io.druid.math.expr.Expr; +import io.druid.math.expr.ExprEval; +import io.druid.math.expr.ExprMacroTable; +import io.druid.query.lookup.LookupReferencesManager; +import io.druid.query.lookup.RegisteredLookupExtractionFn; + +import javax.annotation.Nonnull; +import java.util.List; + +public class LookupExprMacro implements ExprMacroTable.ExprMacro +{ + private final LookupReferencesManager lookupReferencesManager; + + @Inject + public LookupExprMacro(final LookupReferencesManager lookupReferencesManager) + { + this.lookupReferencesManager = lookupReferencesManager; + } + + @Override + public String name() + { + return "lookup"; + } + + @Override + public Expr apply(final List args) + { + if (args.size() != 2) { + throw new IAE("Function[%s] must have 2 arguments", name()); + } + + final Expr arg = args.get(0); + final Expr lookupExpr = args.get(1); + + if (!lookupExpr.isLiteral() || lookupExpr.getLiteralValue() == null) { + throw new IAE("Function[%s] second argument must be a registered lookup name", name()); + } + + final String lookupName = lookupExpr.getLiteralValue().toString(); + final RegisteredLookupExtractionFn extractionFn = new RegisteredLookupExtractionFn( + lookupReferencesManager, + lookupName, + false, + null, + false, + null + ); + + class LookupExpr implements Expr + { + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + return ExprEval.of(extractionFn.apply(Strings.emptyToNull(arg.eval(bindings).asString()))); + } + + @Override + public void visit(final Visitor visitor) + { + arg.visit(visitor); + visitor.visit(this); + } + } + + return new LookupExpr(); + } +} diff --git a/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java b/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java new file mode 100644 index 00000000000..f9a4273a05a --- /dev/null +++ b/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java @@ -0,0 +1,80 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.expression; + +import com.google.common.base.Strings; +import io.druid.java.util.common.IAE; +import io.druid.math.expr.Expr; +import io.druid.math.expr.ExprEval; +import io.druid.math.expr.ExprMacroTable; + +import javax.annotation.Nonnull; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class RegexpExtractExprMacro implements ExprMacroTable.ExprMacro +{ + @Override + public String name() + { + return "regexp_extract"; + } + + @Override + public Expr apply(final List args) + { + if (args.size() < 2 || args.size() > 3) { + throw new IAE("Function[%s] must have 2 to 3 arguments", name()); + } + + final Expr arg = args.get(0); + final Expr patternExpr = args.get(1); + final Expr indexExpr = args.size() > 2 ? args.get(2) : null; + + if (!patternExpr.isLiteral() || (indexExpr != null && !indexExpr.isLiteral())) { + throw new IAE("Function[%s] pattern and index must be literals", name()); + } + + // Precompile the pattern. + final Pattern pattern = Pattern.compile(String.valueOf(patternExpr.getLiteralValue())); + + final int index = indexExpr == null ? 0 : ((Number) indexExpr.getLiteralValue()).intValue(); + class RegexpExtractExpr implements Expr + { + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + final Matcher matcher = pattern.matcher(Strings.nullToEmpty(arg.eval(bindings).asString())); + final String retVal = matcher.find() ? matcher.group(index) : null; + return ExprEval.of(Strings.emptyToNull(retVal)); + } + + @Override + public void visit(final Visitor visitor) + { + arg.visit(visitor); + visitor.visit(this); + } + } + return new RegexpExtractExpr(); + } +} diff --git a/processing/src/main/java/io/druid/query/expression/TimestampCeilExprMacro.java b/processing/src/main/java/io/druid/query/expression/TimestampCeilExprMacro.java new file mode 100644 index 00000000000..301a11bf3d0 --- /dev/null +++ b/processing/src/main/java/io/druid/query/expression/TimestampCeilExprMacro.java @@ -0,0 +1,117 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.expression; + +import io.druid.java.util.common.IAE; +import io.druid.java.util.common.granularity.Granularity; +import io.druid.java.util.common.granularity.PeriodGranularity; +import io.druid.math.expr.Expr; +import io.druid.math.expr.ExprEval; +import io.druid.math.expr.ExprMacroTable; +import org.joda.time.DateTime; + +import javax.annotation.Nonnull; +import java.util.List; + +public class TimestampCeilExprMacro implements ExprMacroTable.ExprMacro +{ + @Override + public String name() + { + return "timestamp_ceil"; + } + + @Override + public Expr apply(final List args) + { + if (args.size() < 2 || args.size() > 4) { + throw new IAE("Function[%s] must have 2 to 4 arguments", name()); + } + + if (args.stream().skip(1).allMatch(Expr::isLiteral)) { + return new TimestampCeilExpr(args); + } else { + return new TimestampCeilDynamicExpr(args); + } + } + + private static class TimestampCeilExpr implements Expr + { + private final Expr arg; + private final Granularity granularity; + + public TimestampCeilExpr(final List args) + { + this.arg = args.get(0); + this.granularity = getGranularity(args, ExprUtils.nilBindings()); + } + + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + return ExprEval.of(granularity.bucketEnd(new DateTime(arg.eval(bindings).asLong())).getMillis()); + } + + @Override + public void visit(final Visitor visitor) + { + arg.visit(visitor); + visitor.visit(this); + } + } + + private static PeriodGranularity getGranularity(final List args, final Expr.ObjectBinding bindings) + { + return ExprUtils.toPeriodGranularity( + args.get(1), + args.size() > 2 ? args.get(2) : null, + args.size() > 3 ? args.get(3) : null, + bindings + ); + } + + private static class TimestampCeilDynamicExpr implements Expr + { + private final List args; + + public TimestampCeilDynamicExpr(final List args) + { + this.args = args; + } + + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + final PeriodGranularity granularity = getGranularity(args, bindings); + return ExprEval.of(granularity.bucketEnd(new DateTime(args.get(0).eval(bindings).asLong())).getMillis()); + } + + @Override + public void visit(final Visitor visitor) + { + for (Expr arg : args) { + arg.visit(visitor); + } + visitor.visit(this); + } + } +} diff --git a/processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java b/processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java new file mode 100644 index 00000000000..261e9caf879 --- /dev/null +++ b/processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java @@ -0,0 +1,129 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.expression; + +import io.druid.java.util.common.IAE; +import io.druid.java.util.common.ISE; +import io.druid.math.expr.Expr; +import io.druid.math.expr.ExprEval; +import io.druid.math.expr.ExprMacroTable; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.chrono.ISOChronology; + +import javax.annotation.Nonnull; +import java.util.List; + +public class TimestampExtractExprMacro implements ExprMacroTable.ExprMacro +{ + public enum Unit + { + EPOCH, + SECOND, + MINUTE, + HOUR, + DAY, + DOW, + DOY, + WEEK, + MONTH, + QUARTER, + YEAR + } + + @Override + public String name() + { + return "timestamp_extract"; + } + + @Override + public Expr apply(final List args) + { + if (args.size() < 2 || args.size() > 3) { + throw new IAE("Function[%s] must have 2 to 3 arguments", name()); + } + + if (!args.get(1).isLiteral() || args.get(1).getLiteralValue() == null) { + throw new IAE("Function[%s] unit arg must be literal", name()); + } + + if (args.size() > 2 && !args.get(2).isLiteral()) { + throw new IAE("Function[%s] timezone arg must be literal", name()); + } + + final Expr arg = args.get(0); + final Unit unit = Unit.valueOf(((String) args.get(1).getLiteralValue()).toUpperCase()); + final DateTimeZone timeZone; + + if (args.size() > 2) { + timeZone = ExprUtils.toTimeZone(args.get(2)); + } else { + timeZone = DateTimeZone.UTC; + } + + final ISOChronology chronology = ISOChronology.getInstance(timeZone); + + class TimestampExtractExpr implements Expr + { + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + final DateTime dateTime = new DateTime(arg.eval(bindings).asLong()).withChronology(chronology); + switch (unit) { + case EPOCH: + return ExprEval.of(dateTime.getMillis()); + case SECOND: + return ExprEval.of(dateTime.secondOfMinute().get()); + case MINUTE: + return ExprEval.of(dateTime.minuteOfHour().get()); + case HOUR: + return ExprEval.of(dateTime.hourOfDay().get()); + case DAY: + return ExprEval.of(dateTime.dayOfMonth().get()); + case DOW: + return ExprEval.of(dateTime.dayOfWeek().get()); + case DOY: + return ExprEval.of(dateTime.dayOfYear().get()); + case WEEK: + return ExprEval.of(dateTime.weekOfWeekyear().get()); + case MONTH: + return ExprEval.of(dateTime.monthOfYear().get()); + case QUARTER: + return ExprEval.of((dateTime.monthOfYear().get() - 1) / 3 + 1); + case YEAR: + return ExprEval.of(dateTime.year().get()); + default: + throw new ISE("Unhandled unit[%s]", unit); + } + } + + @Override + public void visit(final Visitor visitor) + { + arg.visit(visitor); + visitor.visit(this); + } + } + + return new TimestampExtractExpr(); + } +} diff --git a/processing/src/main/java/io/druid/query/expression/TimestampFloorExprMacro.java b/processing/src/main/java/io/druid/query/expression/TimestampFloorExprMacro.java new file mode 100644 index 00000000000..cf660ba346a --- /dev/null +++ b/processing/src/main/java/io/druid/query/expression/TimestampFloorExprMacro.java @@ -0,0 +1,117 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.expression; + +import io.druid.java.util.common.IAE; +import io.druid.java.util.common.granularity.Granularity; +import io.druid.java.util.common.granularity.PeriodGranularity; +import io.druid.math.expr.Expr; +import io.druid.math.expr.ExprEval; +import io.druid.math.expr.ExprMacroTable; +import org.joda.time.DateTime; + +import javax.annotation.Nonnull; +import java.util.List; + +public class TimestampFloorExprMacro implements ExprMacroTable.ExprMacro +{ + @Override + public String name() + { + return "timestamp_floor"; + } + + @Override + public Expr apply(final List args) + { + if (args.size() < 2 || args.size() > 4) { + throw new IAE("Function[%s] must have 2 to 4 arguments", name()); + } + + if (args.stream().skip(1).allMatch(Expr::isLiteral)) { + return new TimestampFloorExpr(args); + } else { + return new TimestampFloorDynamicExpr(args); + } + } + + private static PeriodGranularity getGranularity(final List args, final Expr.ObjectBinding bindings) + { + return ExprUtils.toPeriodGranularity( + args.get(1), + args.size() > 2 ? args.get(2) : null, + args.size() > 3 ? args.get(3) : null, + bindings + ); + } + + private static class TimestampFloorExpr implements Expr + { + private final Expr arg; + private final Granularity granularity; + + public TimestampFloorExpr(final List args) + { + this.arg = args.get(0); + this.granularity = getGranularity(args, ExprUtils.nilBindings()); + } + + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + return ExprEval.of(granularity.bucketStart(new DateTime(arg.eval(bindings).asLong())).getMillis()); + } + + @Override + public void visit(final Visitor visitor) + { + arg.visit(visitor); + visitor.visit(this); + } + } + + private static class TimestampFloorDynamicExpr implements Expr + { + private final List args; + + public TimestampFloorDynamicExpr(final List args) + { + this.args = args; + } + + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + final PeriodGranularity granularity = getGranularity(args, bindings); + return ExprEval.of(granularity.bucketStart(new DateTime(args.get(0).eval(bindings).asLong())).getMillis()); + } + + @Override + public void visit(final Visitor visitor) + { + for (Expr arg : args) { + arg.visit(visitor); + } + visitor.visit(this); + } + } +} diff --git a/processing/src/main/java/io/druid/query/expression/TimestampFormatExprMacro.java b/processing/src/main/java/io/druid/query/expression/TimestampFormatExprMacro.java new file mode 100644 index 00000000000..71af26d8633 --- /dev/null +++ b/processing/src/main/java/io/druid/query/expression/TimestampFormatExprMacro.java @@ -0,0 +1,90 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.expression; + +import com.google.common.base.Preconditions; +import io.druid.java.util.common.IAE; +import io.druid.math.expr.Expr; +import io.druid.math.expr.ExprEval; +import io.druid.math.expr.ExprMacroTable; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; + +import javax.annotation.Nonnull; +import java.util.List; + +public class TimestampFormatExprMacro implements ExprMacroTable.ExprMacro +{ + @Override + public String name() + { + return "timestamp_format"; + } + + @Override + public Expr apply(final List args) + { + if (args.size() < 1 || args.size() > 3) { + throw new IAE("Function[%s] must have 1 to 3 arguments", name()); + } + + final Expr arg = args.get(0); + final String formatString; + final DateTimeZone timeZone; + + if (args.size() > 1) { + Preconditions.checkArgument(args.get(1).isLiteral(), "Function[%s] format arg must be a literal", name()); + formatString = (String) args.get(1).getLiteralValue(); + } else { + formatString = null; + } + + if (args.size() > 2) { + timeZone = ExprUtils.toTimeZone(args.get(2)); + } else { + timeZone = DateTimeZone.UTC; + } + + final DateTimeFormatter formatter = formatString == null + ? ISODateTimeFormat.dateTime() + : DateTimeFormat.forPattern(formatString).withZone(timeZone); + + class TimestampFormatExpr implements Expr + { + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + return ExprEval.of(formatter.print(arg.eval(bindings).asLong())); + } + + @Override + public void visit(final Visitor visitor) + { + arg.visit(visitor); + visitor.visit(this); + } + } + + return new TimestampFormatExpr(); + } +} diff --git a/processing/src/main/java/io/druid/query/expression/TimestampParseExprMacro.java b/processing/src/main/java/io/druid/query/expression/TimestampParseExprMacro.java new file mode 100644 index 00000000000..b3a5ec829eb --- /dev/null +++ b/processing/src/main/java/io/druid/query/expression/TimestampParseExprMacro.java @@ -0,0 +1,89 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.expression; + +import io.druid.java.util.common.IAE; +import io.druid.math.expr.Expr; +import io.druid.math.expr.ExprEval; +import io.druid.math.expr.ExprMacroTable; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; + +import javax.annotation.Nonnull; +import java.util.List; + +public class TimestampParseExprMacro implements ExprMacroTable.ExprMacro +{ + @Override + public String name() + { + return "timestamp_parse"; + } + + @Override + public Expr apply(final List args) + { + if (args.size() < 1 || args.size() > 3) { + throw new IAE("Function[%s] must have 1 to 3 arguments", name()); + } + + final Expr arg = args.get(0); + final String formatString = args.size() > 1 ? (String) args.get(1).getLiteralValue() : null; + final DateTimeZone timeZone; + + if (args.size() > 2 && args.get(2).getLiteralValue() != null) { + timeZone = DateTimeZone.forID((String) args.get(2).getLiteralValue()); + } else { + timeZone = DateTimeZone.UTC; + } + + final DateTimeFormatter formatter = formatString == null + ? ISODateTimeFormat.dateTimeParser() + : DateTimeFormat.forPattern(formatString).withZone(timeZone); + + class TimestampParseExpr implements Expr + { + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + try { + return ExprEval.of(formatter.parseDateTime(arg.eval(bindings).asString()).getMillis()); + } + catch (IllegalArgumentException e) { + // Catch exceptions potentially thrown by formatter.parseDateTime. Our docs say that unparseable timestamps + // are returned as nulls. + return ExprEval.of(null); + } + } + + @Override + public void visit(final Visitor visitor) + { + arg.visit(visitor); + visitor.visit(this); + } + } + + return new TimestampParseExpr(); + } +} diff --git a/processing/src/main/java/io/druid/query/expression/TimestampShiftExprMacro.java b/processing/src/main/java/io/druid/query/expression/TimestampShiftExprMacro.java new file mode 100644 index 00000000000..2bbbed90e78 --- /dev/null +++ b/processing/src/main/java/io/druid/query/expression/TimestampShiftExprMacro.java @@ -0,0 +1,133 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.expression; + +import io.druid.java.util.common.IAE; +import io.druid.java.util.common.granularity.PeriodGranularity; +import io.druid.math.expr.Expr; +import io.druid.math.expr.ExprEval; +import io.druid.math.expr.ExprMacroTable; +import org.joda.time.Chronology; +import org.joda.time.Period; +import org.joda.time.chrono.ISOChronology; + +import javax.annotation.Nonnull; +import java.util.List; + +public class TimestampShiftExprMacro implements ExprMacroTable.ExprMacro +{ + @Override + public String name() + { + return "timestamp_shift"; + } + + @Override + public Expr apply(final List args) + { + if (args.size() < 3 || args.size() > 4) { + throw new IAE("Function[%s] must have 3 to 4 arguments", name()); + } + + if (args.stream().skip(1).allMatch(Expr::isLiteral)) { + return new TimestampShiftExpr(args); + } else { + // Use dynamic impl if any args are non-literal. Don't bother optimizing for the case where period is + // literal but step isn't. + return new TimestampShiftDynamicExpr(args); + } + } + + private static PeriodGranularity getGranularity(final List args, final Expr.ObjectBinding bindings) + { + return ExprUtils.toPeriodGranularity( + args.get(1), + null, + args.size() > 3 ? args.get(3) : null, + bindings + ); + } + + private static int getStep(final List args, final Expr.ObjectBinding bindings) + { + return args.get(2).eval(bindings).asInt(); + } + + private static class TimestampShiftExpr implements Expr + { + private final Expr arg; + private final Chronology chronology; + private final Period period; + private final int step; + + public TimestampShiftExpr(final List args) + { + final PeriodGranularity granularity = getGranularity(args, ExprUtils.nilBindings()); + arg = args.get(0); + period = granularity.getPeriod(); + chronology = ISOChronology.getInstance(granularity.getTimeZone()); + step = getStep(args, ExprUtils.nilBindings()); + } + + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + return ExprEval.of(chronology.add(period, arg.eval(bindings).asLong(), step)); + } + + @Override + public void visit(final Visitor visitor) + { + arg.visit(visitor); + visitor.visit(this); + } + } + + private static class TimestampShiftDynamicExpr implements Expr + { + private final List args; + + public TimestampShiftDynamicExpr(final List args) + { + this.args = args; + } + + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + final PeriodGranularity granularity = getGranularity(args, bindings); + final Period period = granularity.getPeriod(); + final Chronology chronology = ISOChronology.getInstance(granularity.getTimeZone()); + final int step = getStep(args, bindings); + return ExprEval.of(chronology.add(period, args.get(0).eval(bindings).asLong(), step)); + } + + @Override + public void visit(final Visitor visitor) + { + for (Expr arg : args) { + arg.visit(visitor); + } + visitor.visit(this); + } + } +} diff --git a/processing/src/test/java/io/druid/query/expression/ExprMacroTest.java b/processing/src/test/java/io/druid/query/expression/ExprMacroTest.java new file mode 100644 index 00000000000..5c09723c8a9 --- /dev/null +++ b/processing/src/test/java/io/druid/query/expression/ExprMacroTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.expression; + +import com.google.common.collect.ImmutableMap; +import io.druid.math.expr.Expr; +import io.druid.math.expr.Parser; +import org.joda.time.DateTime; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class ExprMacroTest +{ + private static final Expr.ObjectBinding BINDINGS = Parser.withMap( + ImmutableMap.builder() + .put("t", new DateTime("2000-02-03T04:05:06").getMillis()) + .put("tstr", "2000-02-03T04:05:06") + .put("tstr_sql", "2000-02-03 04:05:06") + .put("x", "foo") + .put("y", 2) + .put("z", 3.1) + .put("CityOfAngels", "America/Los_Angeles") + .build() + ); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void testLike() + { + assertExpr("like(x, 'f%')", 1L); + assertExpr("like(x, 'f__')", 1L); + assertExpr("like(x, '%o%')", 1L); + assertExpr("like(x, 'b%')", 0L); + assertExpr("like(x, 'b__')", 0L); + assertExpr("like(x, '%x%')", 0L); + assertExpr("like(x, '')", 0L); + } + + @Test + public void testLookup() + { + assertExpr("lookup(x, 'lookyloo')", "xfoo"); + } + + @Test + public void testLookupNotFound() + { + expectedException.expect(NullPointerException.class); + expectedException.expectMessage("Lookup [lookylook] not found"); + assertExpr("lookup(x, 'lookylook')", null); + } + + @Test + public void testRegexpExtract() + { + assertExpr("regexp_extract(x, 'f(.)')", "fo"); + assertExpr("regexp_extract(x, 'f(.)', 0)", "fo"); + assertExpr("regexp_extract(x, 'f(.)', 1)", "o"); + } + + @Test + public void testTimestampCeil() + { + assertExpr("timestamp_ceil(t, 'P1M')", new DateTime("2000-03-01").getMillis()); + assertExpr("timestamp_ceil(t, 'P1D','','America/Los_Angeles')", new DateTime("2000-02-03T08").getMillis()); + assertExpr("timestamp_ceil(t, 'P1D','',CityOfAngels)", new DateTime("2000-02-03T08").getMillis()); + assertExpr("timestamp_ceil(t, 'P1D','1970-01-01T01','Etc/UTC')", new DateTime("2000-02-04T01").getMillis()); + } + + @Test + public void testTimestampFloor() + { + assertExpr("timestamp_floor(t, 'P1M')", new DateTime("2000-02-01").getMillis()); + assertExpr("timestamp_floor(t, 'P1D','','America/Los_Angeles')", new DateTime("2000-02-02T08").getMillis()); + assertExpr("timestamp_floor(t, 'P1D','',CityOfAngels)", new DateTime("2000-02-02T08").getMillis()); + assertExpr("timestamp_floor(t, 'P1D','1970-01-01T01','Etc/UTC')", new DateTime("2000-02-03T01").getMillis()); + } + + @Test + public void testTimestampShift() + { + assertExpr("timestamp_shift(t, 'P1D', 2)", new DateTime("2000-02-05T04:05:06").getMillis()); + assertExpr("timestamp_shift(t, 'P1D', 2, 'America/Los_Angeles')", new DateTime("2000-02-05T04:05:06").getMillis()); + assertExpr("timestamp_shift(t, 'P1D', 2, CityOfAngels)", new DateTime("2000-02-05T04:05:06").getMillis()); + assertExpr("timestamp_shift(t, 'P1D', 2, '-08:00')", new DateTime("2000-02-05T04:05:06").getMillis()); + } + + @Test + public void testTimestampExtract() + { + assertExpr("timestamp_extract(t, 'DAY')", 3L); + assertExpr("timestamp_extract(t, 'HOUR')", 4L); + assertExpr("timestamp_extract(t, 'DAY', 'America/Los_Angeles')", 2L); + assertExpr("timestamp_extract(t, 'HOUR', 'America/Los_Angeles')", 20L); + } + + @Test + public void testTimestampParse() + { + assertExpr("timestamp_parse(tstr)", new DateTime("2000-02-03T04:05:06").getMillis()); + assertExpr("timestamp_parse(tstr_sql)", null); + assertExpr("timestamp_parse(tstr_sql,'yyyy-MM-dd HH:mm:ss')", new DateTime("2000-02-03T04:05:06").getMillis()); + assertExpr( + "timestamp_parse(tstr_sql,'yyyy-MM-dd HH:mm:ss','America/Los_Angeles')", + new DateTime("2000-02-03T04:05:06-08:00").getMillis() + ); + } + + @Test + public void testTimestampFormat() + { + assertExpr("timestamp_format(t)", "2000-02-03T04:05:06.000Z"); + assertExpr("timestamp_format(t,'yyyy-MM-dd HH:mm:ss')", "2000-02-03 04:05:06"); + assertExpr("timestamp_format(t,'yyyy-MM-dd HH:mm:ss','America/Los_Angeles')", "2000-02-02 20:05:06"); + } + + private void assertExpr(final String expression, final Object expectedResult) + { + final Expr expr = Parser.parse(expression, TestExprMacroTable.INSTANCE); + Assert.assertEquals(expression, expectedResult, expr.eval(BINDINGS).value()); + } +} diff --git a/processing/src/test/java/io/druid/query/expression/TestExprMacroTable.java b/processing/src/test/java/io/druid/query/expression/TestExprMacroTable.java index 89661ac747a..4321a03f119 100644 --- a/processing/src/test/java/io/druid/query/expression/TestExprMacroTable.java +++ b/processing/src/test/java/io/druid/query/expression/TestExprMacroTable.java @@ -20,7 +20,17 @@ package io.druid.query.expression; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import io.druid.math.expr.ExprMacroTable; +import io.druid.query.extraction.MapLookupExtractor; +import io.druid.query.lookup.LookupExtractor; +import io.druid.query.lookup.LookupExtractorFactory; +import io.druid.query.lookup.LookupExtractorFactoryContainer; +import io.druid.query.lookup.LookupIntrospectHandler; +import io.druid.query.lookup.LookupReferencesManager; +import org.easymock.EasyMock; + +import javax.annotation.Nullable; public class TestExprMacroTable extends ExprMacroTable { @@ -30,8 +40,64 @@ public class TestExprMacroTable extends ExprMacroTable { super( ImmutableList.of( - new LikeExprMacro() + new LikeExprMacro(), + new LookupExprMacro(createTestLookupReferencesManager(ImmutableMap.of("foo", "xfoo"))), + new RegexpExtractExprMacro(), + new TimestampCeilExprMacro(), + new TimestampExtractExprMacro(), + new TimestampFloorExprMacro(), + new TimestampFormatExprMacro(), + new TimestampParseExprMacro(), + new TimestampShiftExprMacro() ) ); } + + /** + * Returns a mock {@link LookupReferencesManager} that has one lookup, "lookyloo". + */ + public static LookupReferencesManager createTestLookupReferencesManager(final ImmutableMap theLookup) + { + final LookupReferencesManager lookupReferencesManager = EasyMock.createMock(LookupReferencesManager.class); + EasyMock.expect(lookupReferencesManager.get(EasyMock.eq("lookyloo"))).andReturn( + new LookupExtractorFactoryContainer( + "v0", + new LookupExtractorFactory() + { + @Override + public boolean start() + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean close() + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean replaces(@Nullable final LookupExtractorFactory other) + { + throw new UnsupportedOperationException(); + } + + @Override + public LookupIntrospectHandler getIntrospectHandler() + { + throw new UnsupportedOperationException(); + } + + @Override + public LookupExtractor get() + { + return new MapLookupExtractor(theLookup, false); + } + } + ) + ).anyTimes(); + EasyMock.expect(lookupReferencesManager.get(EasyMock.not(EasyMock.eq("lookyloo")))).andReturn(null).anyTimes(); + EasyMock.replay(lookupReferencesManager); + return lookupReferencesManager; + } } diff --git a/server/src/main/java/io/druid/guice/ExpressionModule.java b/server/src/main/java/io/druid/guice/ExpressionModule.java index 100e5329748..d9c1da61bad 100644 --- a/server/src/main/java/io/druid/guice/ExpressionModule.java +++ b/server/src/main/java/io/druid/guice/ExpressionModule.java @@ -27,6 +27,13 @@ import io.druid.initialization.DruidModule; import io.druid.math.expr.ExprMacroTable; import io.druid.query.expression.GuiceExprMacroTable; import io.druid.query.expression.LikeExprMacro; +import io.druid.query.expression.RegexpExtractExprMacro; +import io.druid.query.expression.TimestampCeilExprMacro; +import io.druid.query.expression.TimestampExtractExprMacro; +import io.druid.query.expression.TimestampFloorExprMacro; +import io.druid.query.expression.TimestampFormatExprMacro; +import io.druid.query.expression.TimestampParseExprMacro; +import io.druid.query.expression.TimestampShiftExprMacro; import java.util.List; @@ -37,6 +44,13 @@ public class ExpressionModule implements DruidModule public static final List> EXPR_MACROS = ImmutableList.>builder() .add(LikeExprMacro.class) + .add(RegexpExtractExprMacro.class) + .add(TimestampCeilExprMacro.class) + .add(TimestampExtractExprMacro.class) + .add(TimestampFloorExprMacro.class) + .add(TimestampFormatExprMacro.class) + .add(TimestampParseExprMacro.class) + .add(TimestampShiftExprMacro.class) .build(); @Override diff --git a/server/src/main/java/io/druid/query/lookup/LookupModule.java b/server/src/main/java/io/druid/query/lookup/LookupModule.java index 267b4bf1393..6932822fa0d 100644 --- a/server/src/main/java/io/druid/query/lookup/LookupModule.java +++ b/server/src/main/java/io/druid/query/lookup/LookupModule.java @@ -35,6 +35,7 @@ import com.google.inject.Binder; import com.google.inject.Inject; import io.druid.common.utils.ServletResourceUtils; import io.druid.curator.announcement.Announcer; +import io.druid.guice.ExpressionModule; import io.druid.guice.Jerseys; import io.druid.guice.JsonConfigProvider; import io.druid.guice.LifecycleModule; @@ -44,6 +45,7 @@ import io.druid.guice.annotations.Self; import io.druid.guice.annotations.Smile; import io.druid.initialization.DruidModule; import io.druid.java.util.common.logger.Logger; +import io.druid.query.expression.LookupExprMacro; import io.druid.server.DruidNode; import io.druid.server.initialization.ZkPathsConfig; import io.druid.server.initialization.jetty.JettyBindings; @@ -89,6 +91,7 @@ public class LookupModule implements DruidModule JsonConfigProvider.bind(binder, PROPERTY_BASE, LookupListeningAnnouncerConfig.class); Jerseys.addResource(binder, LookupListeningResource.class); Jerseys.addResource(binder, LookupIntrospectionResource.class); + ExpressionModule.addExprMacro(binder, LookupExprMacro.class); LifecycleModule.register(binder, LookupResourceListenerAnnouncer.class); // Nothing else starts this, so we bind it to get it to start binder.bind(LookupResourceListenerAnnouncer.class).in(ManageLifecycle.class);