Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

string -> expression -> string -> expression (#9367) 

* add Expr.stringify which produces parseable expression strings, parser support for null values in arrays, and parser support for empty numeric arrays

* oops, macros are expressions too

* style

* spotbugs

* qualified type arrays

* review stuffs

* simplify grammar

* more permissive array parsing

* reuse expr joiner

* fix it
This commit is contained in:
Clint Wylie 2020-02-21 15:43:02 -08:00 committed by GitHub
parent 05258dca37
commit 6d8dd5ec10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 750 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
core/src/main/antlr4/org/apache/druid/math/expr/antlr/Expr.g4
View File

@ -15,32 +15,42 @@
grammar Expr;
expr : 'null' # null
| ('-'|'!') expr # unaryOpExpr
|<assoc=right> expr '^' expr # powOpExpr
| expr ('*'|'/'|'%') expr # mulDivModuloExpr
| expr ('+'|'-') expr # addSubExpr
| expr ('<'|'<='|'>'|'>='|'=='|'!=') expr # logicalOpExpr
| expr ('&&'|'||') expr # logicalAndOrExpr
| '(' expr ')' # nestedExpr
| IDENTIFIER '(' lambda ',' fnArgs ')' # applyFunctionExpr
| IDENTIFIER '(' fnArgs? ')' # functionExpr
| IDENTIFIER # identifierExpr
| DOUBLE # doubleExpr
| LONG # longExpr
| STRING # string
| '[' DOUBLE (',' DOUBLE)* ']' # doubleArray
| '[' LONG (',' LONG)* ']' # longArray
| '[' STRING (',' STRING)* ']' # stringArray
| '[]' # emptyArray
expr : NULL # null
| ('-'|'!') expr # unaryOpExpr
|<assoc=right> expr '^' expr # powOpExpr
| expr ('*'|'/'|'%') expr # mulDivModuloExpr
| expr ('+'|'-') expr # addSubExpr
| expr ('<'|'<='|'>'|'>='|'=='|'!=') expr # logicalOpExpr
| expr ('&&'|'||') expr # logicalAndOrExpr
| '(' expr ')' # nestedExpr
| IDENTIFIER '(' lambda ',' fnArgs ')' # applyFunctionExpr
| IDENTIFIER '(' fnArgs? ')' # functionExpr
| IDENTIFIER # identifierExpr
| DOUBLE # doubleExpr
| LONG # longExpr
| STRING # string
| '[' (stringElement (',' stringElement)*)? ']' # stringArray
| '[' longElement (',' longElement)*']' # longArray
| '<LONG>' '[' (numericElement (',' numericElement)*)? ']' # explicitLongArray
| '<DOUBLE>'? '[' (numericElement (',' numericElement)*)? ']' # doubleArray
| '<STRING>' '[' (literalElement (',' literalElement)*)? ']' # explicitStringArray
;
lambda : (IDENTIFIER | '(' ')' | '(' IDENTIFIER (',' IDENTIFIER)* ')') '->' expr
;
fnArgs : expr (',' expr)* # functionArgs
fnArgs : expr (',' expr)* # functionArgs
;
stringElement : (STRING | NULL);
longElement : (LONG | NULL);
numericElement : (LONG | DOUBLE | NULL);
literalElement : (STRING | LONG | DOUBLE | NULL);
NULL : 'null';
IDENTIFIER : [_$a-zA-Z][_$a-zA-Z0-9]* | '"' (ESC | ~ [\"\\])* '"';
LONG : [0-9]+ ;
DOUBLE : [0-9]+ '.' [0-9]* ;

40
core/src/main/java/org/apache/druid/java/util/common/Numbers.java
View File

@ -118,6 +118,7 @@ public final class Numbers
}
}
/**
* Try parsing the given Number or String object val as long.
* @param val
@ -167,6 +168,45 @@ public final class Numbers
}
}
/**
* Like {@link #tryParseDouble}, but does not produce a primitive and will explode if unable to produce a Double
* similar to {@link Double#parseDouble}
*/
@Nullable
public static Double parseDoubleObject(@Nullable String val)
{
if (val == null) {
return null;
}
Double d = Doubles.tryParse(val);
if (d != null) {
return d;
}
throw new NumberFormatException("Cannot parse string to double");
}
/**
* Like {@link #tryParseLong} but does not produce a primitive and will explode if unable to produce a Long
* similar to {@link Long#parseLong}
*/
@Nullable
public static Long parseLongObject(@Nullable String val)
{
if (val == null) {
return null;
}
Long lobj = Longs.tryParse(val);
if (lobj != null) {
return lobj;
}
// try as a double, for "ddd.dd" , Longs.tryParse(..) returns null
Double dobj = Doubles.tryParse((String) val);
if (dobj != null) {
return dobj.longValue();
}
throw new NumberFormatException("Cannot parse string to long");
}
public static int toIntExact(long value, String error)
{
if ((int) value != value) {

112
core/src/main/java/org/apache/druid/math/expr/Expr.java
View File

@ -19,12 +19,14 @@
package org.apache.druid.math.expr;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.common.math.LongMath;
import com.google.common.primitives.Ints;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.ISE;
@ -46,6 +48,8 @@ import java.util.stream.Collectors;
*/
public interface Expr
{
String NULL_LITERAL = "null";
Joiner ARG_JOINER = Joiner.on(", ");
/**
* Indicates expression is a constant whose literal value can be extracted by {@link Expr#getLiteralValue()},
* making evaluating with arguments and bindings unecessary
@ -109,6 +113,12 @@ public interface Expr
*/
ExprEval eval(ObjectBinding bindings);
/**
* Convert the {@link Expr} back into parseable string that when parsed with
* {@link Parser#parse(String, ExprMacroTable)} will produce an equivalent {@link Expr}.
*/
String stringify();
/**
* Programmatically inspect the {@link Expr} tree with a {@link Visitor}. Each {@link Expr} is responsible for
* ensuring the {@link Visitor} can visit all of its {@link Expr} children before visiting itself
@ -463,6 +473,12 @@ abstract class ConstantExpr implements Expr
{
return new BindingDetails();
}
@Override
public String stringify()
{
return toString();
}
}
abstract class NullNumericConstantExpr extends ConstantExpr
@ -476,7 +492,7 @@ abstract class NullNumericConstantExpr extends ConstantExpr
@Override
public String toString()
{
return "null";
return NULL_LITERAL;
}
}
@ -544,6 +560,15 @@ class LongArrayExpr extends ConstantExpr
{
return ExprEval.ofLongArray(value);
}
@Override
public String stringify()
{
if (value.length == 0) {
return "<LONG>[]";
}
return StringUtils.format("<LONG>%s", toString());
}
}
class StringExpr extends ConstantExpr
@ -574,6 +599,13 @@ class StringExpr extends ConstantExpr
{
return ExprEval.of(value);
}
@Override
public String stringify()
{
// escape as javascript string since string literals are wrapped in single quotes
return value == null ? NULL_LITERAL : StringUtils.format("'%s'", StringEscapeUtils.escapeJavaScript(value));
}
}
class StringArrayExpr extends ConstantExpr
@ -602,6 +634,27 @@ class StringArrayExpr extends ConstantExpr
{
return ExprEval.ofStringArray(value);
}
@Override
public String stringify()
{
if (value.length == 0) {
return "<STRING>[]";
}
return StringUtils.format(
"<STRING>[%s]",
ARG_JOINER.join(
Arrays.stream(value)
.map(s -> s == null
? NULL_LITERAL
// escape as javascript string since string literals are wrapped in single quotes
: StringUtils.format("'%s'", StringEscapeUtils.escapeJavaScript(s))
)
.iterator()
)
);
}
}
class DoubleExpr extends ConstantExpr
@ -667,6 +720,15 @@ class DoubleArrayExpr extends ConstantExpr
{
return ExprEval.ofDoubleArray(value);
}
@Override
public String stringify()
{
if (value.length == 0) {
return "<DOUBLE>[]";
}
return StringUtils.format("<DOUBLE>%s", toString());
}
}
/**
@ -757,6 +819,13 @@ class IdentifierExpr implements Expr
return ExprEval.bestEffortOf(bindings.get(binding));
}
@Override
public String stringify()
{
// escape as java strings since identifiers are wrapped in double quotes
return StringUtils.format("\"%s\"", StringEscapeUtils.escapeJava(binding));
}
@Override
public void visit(Visitor visitor)
{
@ -823,6 +892,12 @@ class LambdaExpr implements Expr
return expr.eval(bindings);
}
@Override
public String stringify()
{
return StringUtils.format("(%s) -> %s", ARG_JOINER.join(getIdentifiers()), expr.stringify());
}
@Override
public void visit(Visitor visitor)
{
@ -879,6 +954,12 @@ class FunctionExpr implements Expr
return function.apply(args, bindings);
}
@Override
public String stringify()
{
return StringUtils.format("%s(%s)", name, ARG_JOINER.join(args.stream().map(Expr::stringify).iterator()));
}
@Override
public void visit(Visitor visitor)
{
@ -965,6 +1046,17 @@ class ApplyFunctionExpr implements Expr
return function.apply(lambdaExpr, argsExpr, bindings);
}
@Override
public String stringify()
{
return StringUtils.format(
"%s(%s, %s)",
name,
lambdaExpr.stringify(),
ARG_JOINER.join(argsExpr.stream().map(Expr::stringify).iterator())
);
}
@Override
public void visit(Visitor visitor)
{
@ -1059,6 +1151,12 @@ class UnaryMinusExpr extends UnaryExpr
throw new IAE("unsupported type " + ret.type());
}
@Override
public String stringify()
{
return StringUtils.format("-%s", expr.stringify());
}
@Override
public String toString()
{
@ -1091,6 +1189,12 @@ class UnaryNotExpr extends UnaryExpr
return ExprEval.of(!ret.asBoolean(), retType);
}
@Override
public String stringify()
{
return StringUtils.format("!%s", expr.stringify());
}
@Override
public String toString()
{
@ -1144,6 +1248,12 @@ abstract class BinaryOpExprBase implements Expr
return StringUtils.format("(%s %s %s)", op, left, right);
}
@Override
public String stringify()
{
return StringUtils.format("(%s %s %s)", left.stringify(), op, right.stringify());
}
protected abstract BinaryOpExprBase copy(Expr left, Expr right);
@Override

116
core/src/main/java/org/apache/druid/math/expr/ExprListenerImpl.java
View File

@ -23,11 +23,13 @@ import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.druid.annotations.UsedInGeneratedCode;
import org.apache.druid.java.util.common.Numbers;
import org.apache.druid.java.util.common.RE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.antlr.ExprBaseListener;
import org.apache.druid.math.expr.antlr.ExprParser;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@ -77,7 +79,7 @@ public class ExprListenerImpl extends ExprBaseListener
nodes.put(ctx, new UnaryNotExpr((Expr) nodes.get(ctx.getChild(1))));
break;
default:
throw new RuntimeException("Unrecognized unary operator " + ctx.getChild(0).getText());
throw new RE("Unrecognized unary operator %s", ctx.getChild(0).getText());
}
}
@ -97,6 +99,7 @@ public class ExprListenerImpl extends ExprBaseListener
);
}
@Override
public void exitDoubleExpr(ExprParser.DoubleExprContext ctx)
{
@ -106,15 +109,6 @@ public class ExprListenerImpl extends ExprBaseListener
);
}
@Override
public void exitDoubleArray(ExprParser.DoubleArrayContext ctx)
{
Double[] values = new Double[ctx.DOUBLE().size()];
for (int i = 0; i < values.length; i++) {
values[i] = Double.parseDouble(ctx.DOUBLE(i).getText());
}
nodes.put(ctx, new DoubleArrayExpr(values));
}
@Override
public void exitAddSubExpr(ExprParser.AddSubExprContext ctx)
@ -142,7 +136,7 @@ public class ExprListenerImpl extends ExprBaseListener
);
break;
default:
throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText());
throw new RE("Unrecognized binary operator %s", ctx.getChild(1).getText());
}
}
@ -181,20 +175,10 @@ public class ExprListenerImpl extends ExprBaseListener
);
break;
default:
throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText());
throw new RE("Unrecognized binary operator %s", ctx.getChild(1).getText());
}
}
@Override
public void exitLongArray(ExprParser.LongArrayContext ctx)
{
Long[] values = new Long[ctx.LONG().size()];
for (int i = 0; i < values.length; i++) {
values[i] = Long.parseLong(ctx.LONG(i).getText());
}
nodes.put(ctx, new LongArrayExpr(values));
}
@Override
public void exitNestedExpr(ExprParser.NestedExprContext ctx)
{
@ -273,7 +257,7 @@ public class ExprListenerImpl extends ExprBaseListener
);
break;
default:
throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText());
throw new RE("Unrecognized binary operator %s", ctx.getChild(1).getText());
}
}
@ -313,7 +297,7 @@ public class ExprListenerImpl extends ExprBaseListener
);
break;
default:
throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText());
throw new RE("Unrecognized binary operator %s", ctx.getChild(1).getText());
}
}
@ -403,20 +387,92 @@ public class ExprListenerImpl extends ExprBaseListener
nodes.put(ctx, new StringExpr(null));
}
@Override
public void exitDoubleArray(ExprParser.DoubleArrayContext ctx)
{
Double[] values = new Double[ctx.numericElement().size()];
for (int i = 0; i < values.length; i++) {
if (ctx.numericElement(i).NULL() != null) {
values[i] = null;
} else if (ctx.numericElement(i).LONG() != null) {
values[i] = Numbers.parseDoubleObject(ctx.numericElement(i).LONG().getText());
} else if (ctx.numericElement(i).DOUBLE() != null) {
values[i] = Numbers.parseDoubleObject(ctx.numericElement(i).DOUBLE().getText());
} else {
throw new RE("Failed to parse array element %s as a double", ctx.numericElement(i).getText());
}
}
nodes.put(ctx, new DoubleArrayExpr(values));
}
@Override
public void exitLongArray(ExprParser.LongArrayContext ctx)
{
Long[] values = new Long[ctx.longElement().size()];
for (int i = 0; i < values.length; i++) {
if (ctx.longElement(i).NULL() != null) {
values[i] = null;
} else if (ctx.longElement(i).LONG() != null) {
values[i] = Long.parseLong(ctx.longElement(i).LONG().getText());
} else {
throw new RE("Failed to parse array element %s as a long", ctx.longElement(i).getText());
}
}
nodes.put(ctx, new LongArrayExpr(values));
}
@Override
public void exitExplicitLongArray(ExprParser.ExplicitLongArrayContext ctx)
{
Long[] values = new Long[ctx.numericElement().size()];
for (int i = 0; i < values.length; i++) {
if (ctx.numericElement(i).NULL() != null) {
values[i] = null;
} else if (ctx.numericElement(i).LONG() != null) {
values[i] = Numbers.parseLongObject(ctx.numericElement(i).LONG().getText());
} else if (ctx.numericElement(i).DOUBLE() != null) {
values[i] = Numbers.parseLongObject(ctx.numericElement(i).DOUBLE().getText());
} else {
throw new RE("Failed to parse array element %s as a long", ctx.numericElement(i).getText());
}
}
nodes.put(ctx, new LongArrayExpr(values));
}
@Override
public void exitStringArray(ExprParser.StringArrayContext ctx)
{
String[] values = new String[ctx.STRING().size()];
String[] values = new String[ctx.stringElement().size()];
for (int i = 0; i < values.length; i++) {
values[i] = escapeStringLiteral(ctx.STRING(i).getText());
if (ctx.stringElement(i).NULL() != null) {
values[i] = null;
} else if (ctx.stringElement(i).STRING() != null) {
values[i] = escapeStringLiteral(ctx.stringElement(i).STRING().getText());
} else {
throw new RE("Failed to parse array: element %s is not a string", ctx.stringElement(i).getText());
}
}
nodes.put(ctx, new StringArrayExpr(values));
}
@Override
public void exitEmptyArray(ExprParser.EmptyArrayContext ctx)
public void exitExplicitStringArray(ExprParser.ExplicitStringArrayContext ctx)
{
nodes.put(ctx, new StringArrayExpr(new String[0]));
String[] values = new String[ctx.literalElement().size()];
for (int i = 0; i < values.length; i++) {
if (ctx.literalElement(i).NULL() != null) {
values[i] = null;
} else if (ctx.literalElement(i).STRING() != null) {
values[i] = escapeStringLiteral(ctx.literalElement(i).STRING().getText());
} else if (ctx.literalElement(i).DOUBLE() != null) {
values[i] = ctx.literalElement(i).DOUBLE().getText();
} else if (ctx.literalElement(i).LONG() != null) {
values[i] = ctx.literalElement(i).LONG().getText();
} else {
throw new RE("Failed to parse array element %s as a string", ctx.literalElement(i).getText());
}
}
nodes.put(ctx, new StringArrayExpr(values));
}
/**
@ -455,8 +511,12 @@ public class ExprListenerImpl extends ExprBaseListener
/**
* Remove single quote from a string literal, returning unquoted string value
*/
@Nullable
private static String escapeStringLiteral(String text)
{
if (text.equalsIgnoreCase(Expr.NULL_LITERAL)) {
return null;
}
String unquoted = text.substring(1, text.length() - 1);
return unquoted.indexOf('\\') >= 0 ? StringEscapeUtils.unescapeJava(unquoted) : unquoted;
}

24
core/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java
View File

@ -97,13 +97,15 @@ public class ExprMacroTable
*/
public abstract static class BaseScalarUnivariateMacroFunctionExpr implements Expr
{
protected final String name;
protected final Expr arg;
// Use Supplier to memoize values as ExpressionSelectors#makeExprEvalSelector() can make repeated calls for them
private final Supplier<BindingDetails> analyzeInputsSupplier;
public BaseScalarUnivariateMacroFunctionExpr(Expr arg)
public BaseScalarUnivariateMacroFunctionExpr(String name, Expr arg)
{
this.name = name;
this.arg = arg;
analyzeInputsSupplier = Suppliers.memoize(this::supplyAnalyzeInputs);
}
@ -121,6 +123,12 @@ public class ExprMacroTable
return analyzeInputsSupplier.get();
}
@Override
public String stringify()
{
return StringUtils.format("%s(%s)", name, arg.stringify());
}
private BindingDetails supplyAnalyzeInputs()
{
return arg.analyzeInputs().withScalarArguments(ImmutableSet.of(arg));
@ -132,17 +140,29 @@ public class ExprMacroTable
*/
public abstract static class BaseScalarMacroFunctionExpr implements Expr
{
protected final String name;
protected final List<Expr> args;
// Use Supplier to memoize values as ExpressionSelectors#makeExprEvalSelector() can make repeated calls for them
private final Supplier<BindingDetails> analyzeInputsSupplier;
public BaseScalarMacroFunctionExpr(final List<Expr> args)
public BaseScalarMacroFunctionExpr(String name, final List<Expr> args)
{
this.name = name;
this.args = args;
analyzeInputsSupplier = Suppliers.memoize(this::supplyAnalyzeInputs);
}
@Override
public String stringify()
{
return StringUtils.format(
"%s(%s)",
name,
Expr.ARG_JOINER.join(args.stream().map(Expr::stringify).iterator())
);
}
@Override
public void visit(final Visitor visitor)
{

8
core/src/main/java/org/apache/druid/math/expr/Parser.java
View File

@ -108,7 +108,7 @@ public class Parser
}
@VisibleForTesting
static Expr parse(String in, ExprMacroTable macroTable, boolean withFlatten)
public static Expr parse(String in, ExprMacroTable macroTable, boolean withFlatten)
{
ExprLexer lexer = new ExprLexer(new ANTLRInputStream(in));
CommonTokenStream tokens = new CommonTokenStream(lexer);
@ -118,7 +118,11 @@ public class Parser
ParseTreeWalker walker = new ParseTreeWalker();
ExprListenerImpl listener = new ExprListenerImpl(parseTree, macroTable);
walker.walk(listener, parseTree);
return withFlatten ? flatten(listener.getAST()) : listener.getAST();
Expr parsed = listener.getAST();
if (parsed == null) {
throw new RE("Failed to parse expression: %s", in);
}
return withFlatten ? flatten(parsed) : parsed;
}
/**

32
core/src/test/java/org/apache/druid/java/util/common/NumbersTest.java
View File

@ -127,4 +127,36 @@ public class NumbersTest
expectedException.expectMessage(CoreMatchers.startsWith("Unknown type"));
Numbers.parseBoolean(new Object());
}
@Test
public void testParseLongObject()
{
Assert.assertEquals(null, Numbers.parseLongObject(null));
Assert.assertEquals((Long) 1L, Numbers.parseLongObject("1"));
Assert.assertEquals((Long) 32L, Numbers.parseLongObject("32.1243"));
}
@Test
public void testParseLongObjectUnparseable()
{
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("Cannot parse string to long");
Assert.assertEquals((Long) 1337L, Numbers.parseLongObject("'1'"));
}
@Test
public void testParseDoubleObject()
{
Assert.assertEquals(null, Numbers.parseLongObject(null));
Assert.assertEquals((Double) 1.0, Numbers.parseDoubleObject("1"));
Assert.assertEquals((Double) 32.1243, Numbers.parseDoubleObject("32.1243"));
}
@Test
public void testParseDoubleObjectUnparseable()
{
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("Cannot parse string to double");
Assert.assertEquals((Double) 300.0, Numbers.parseDoubleObject("'1.1'"));
}
}

46
core/src/test/java/org/apache/druid/math/expr/ApplyFunctionTest.java
View File

@ -102,7 +102,7 @@ public class ApplyFunctionTest extends InitializedNullHandlingTest
assertExpr("fold((b, acc) -> b * acc, map((b) -> b * 2, filter(b -> b > 3, b)), 1)", 80L);
assertExpr("fold((a, acc) -> concat(a, acc), a, '')", "foobarbazbarfoo");
assertExpr("fold((a, acc) -> array_append(acc, a), a, [])", new String[]{"foo", "bar", "baz", "foobar"});
assertExpr("fold((a, acc) -> array_append(acc, a), b, cast([], 'LONG_ARRAY'))", new Long[]{1L, 2L, 3L, 4L, 5L});
assertExpr("fold((a, acc) -> array_append(acc, a), b, <LONG>[])", new Long[]{1L, 2L, 3L, 4L, 5L});
}
@Test
@ -161,6 +161,16 @@ public class ApplyFunctionTest extends InitializedNullHandlingTest
{
final Expr expr = Parser.parse(expression, ExprMacroTable.nil());
Assert.assertEquals(expression, expectedResult, expr.eval(bindings).value());
final Expr exprNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
final Expr roundTrip = Parser.parse(exprNoFlatten.stringify(), ExprMacroTable.nil());
Assert.assertEquals(expr.stringify(), expectedResult, roundTrip.eval(bindings).value());
final Expr roundTripFlatten = Parser.parse(expr.stringify(), ExprMacroTable.nil());
Assert.assertEquals(expr.stringify(), expectedResult, roundTripFlatten.eval(bindings).value());
Assert.assertEquals(expr.stringify(), roundTrip.stringify());
Assert.assertEquals(expr.stringify(), roundTripFlatten.stringify());
}
private void assertExpr(final String expression, final Object[] expectedResult)
@ -170,6 +180,22 @@ public class ApplyFunctionTest extends InitializedNullHandlingTest
if (expectedResult.length != 0 || result == null || result.length != 0) {
Assert.assertArrayEquals(expression, expectedResult, result);
}
final Expr exprNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
final Expr roundTrip = Parser.parse(exprNoFlatten.stringify(), ExprMacroTable.nil());
final Object[] resultRoundTrip = roundTrip.eval(bindings).asArray();
if (expectedResult.length != 0 || resultRoundTrip == null || resultRoundTrip.length != 0) {
Assert.assertArrayEquals(expr.stringify(), expectedResult, resultRoundTrip);
}
final Expr roundTripFlatten = Parser.parse(expr.stringify(), ExprMacroTable.nil());
final Object[] resultRoundTripFlatten = roundTripFlatten.eval(bindings).asArray();
if (expectedResult.length != 0 || resultRoundTripFlatten == null || resultRoundTripFlatten.length != 0) {
Assert.assertArrayEquals(expr.stringify(), expectedResult, resultRoundTripFlatten);
}
Assert.assertEquals(expr.stringify(), roundTrip.stringify());
Assert.assertEquals(expr.stringify(), roundTripFlatten.stringify());
}
private void assertExpr(final String expression, final Double[] expectedResult)
@ -180,5 +206,23 @@ public class ApplyFunctionTest extends InitializedNullHandlingTest
for (int i = 0; i < result.length; i++) {
Assert.assertEquals(expression, expectedResult[i], result[i], 0.00001); // something is lame somewhere..
}
final Expr exprNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
final Expr roundTrip = Parser.parse(exprNoFlatten.stringify(), ExprMacroTable.nil());
Double[] resultRoundTrip = (Double[]) roundTrip.eval(bindings).value();
Assert.assertEquals(expectedResult.length, resultRoundTrip.length);
for (int i = 0; i < resultRoundTrip.length; i++) {
Assert.assertEquals(expression, expectedResult[i], resultRoundTrip[i], 0.00001);
}
final Expr roundTripFlatten = Parser.parse(expr.stringify(), ExprMacroTable.nil());
Double[] resultRoundTripFlatten = (Double[]) roundTripFlatten.eval(bindings).value();
Assert.assertEquals(expectedResult.length, resultRoundTripFlatten.length);
for (int i = 0; i < resultRoundTripFlatten.length; i++) {
Assert.assertEquals(expression, expectedResult[i], resultRoundTripFlatten[i], 0.00001);
}
Assert.assertEquals(expr.stringify(), roundTrip.stringify());
Assert.assertEquals(expr.stringify(), roundTripFlatten.stringify());
}
}

25
core/src/test/java/org/apache/druid/math/expr/FunctionTest.java
View File

@ -231,7 +231,7 @@ public class FunctionTest extends InitializedNullHandlingTest
assertExpr("array_append([1, 2, 3], 4)", new Long[]{1L, 2L, 3L, 4L});
assertExpr("array_append([1, 2, 3], 'bar')", new Long[]{1L, 2L, 3L, null});
assertExpr("array_append([], 1)", new String[]{"1"});
assertExpr("array_append(cast([], 'LONG_ARRAY'), 1)", new Long[]{1L});
assertExpr("array_append(<LONG>[], 1)", new Long[]{1L});
}
@Test
@ -287,18 +287,39 @@ public class FunctionTest extends InitializedNullHandlingTest
assertExpr("array_prepend(4, [1, 2, 3])", new Long[]{4L, 1L, 2L, 3L});
assertExpr("array_prepend('bar', [1, 2, 3])", new Long[]{null, 1L, 2L, 3L});
assertExpr("array_prepend(1, [])", new String[]{"1"});
assertExpr("array_prepend(1, cast([], 'LONG_ARRAY'))", new Long[]{1L});
assertExpr("array_prepend(1, <LONG>[])", new Long[]{1L});
assertExpr("array_prepend(1, <DOUBLE>[])", new Double[]{1.0});
}
private void assertExpr(final String expression, final Object expectedResult)
{
final Expr expr = Parser.parse(expression, ExprMacroTable.nil());
Assert.assertEquals(expression, expectedResult, expr.eval(bindings).value());
final Expr exprNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
final Expr roundTrip = Parser.parse(exprNoFlatten.stringify(), ExprMacroTable.nil());
Assert.assertEquals(expr.stringify(), expectedResult, roundTrip.eval(bindings).value());
final Expr roundTripFlatten = Parser.parse(expr.stringify(), ExprMacroTable.nil());
Assert.assertEquals(expr.stringify(), expectedResult, roundTripFlatten.eval(bindings).value());
Assert.assertEquals(expr.stringify(), roundTrip.stringify());
Assert.assertEquals(expr.stringify(), roundTripFlatten.stringify());
}
private void assertExpr(final String expression, final Object[] expectedResult)
{
final Expr expr = Parser.parse(expression, ExprMacroTable.nil());
Assert.assertArrayEquals(expression, expectedResult, expr.eval(bindings).asArray());
final Expr exprNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
final Expr roundTrip = Parser.parse(exprNoFlatten.stringify(), ExprMacroTable.nil());
Assert.assertArrayEquals(expression, expectedResult, roundTrip.eval(bindings).asArray());
final Expr roundTripFlatten = Parser.parse(expr.stringify(), ExprMacroTable.nil());
Assert.assertArrayEquals(expression, expectedResult, roundTripFlatten.eval(bindings).asArray());
Assert.assertEquals(expr.stringify(), roundTrip.stringify());
Assert.assertEquals(expr.stringify(), roundTripFlatten.stringify());
}
}

153
core/src/test/java/org/apache/druid/math/expr/ParserTest.java
View File

@ -22,9 +22,12 @@ package org.apache.druid.math.expr;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.druid.java.util.common.RE;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import java.util.Collections;
import java.util.List;
@ -35,6 +38,9 @@ import java.util.Set;
*/
public class ParserTest extends InitializedNullHandlingTest
{
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Test
public void testSimple()
{
@ -186,11 +192,96 @@ public class ParserTest extends InitializedNullHandlingTest
}
@Test
public void testLiteralArrays()
public void testLiteralArraysHomogeneousElements()
{
validateConstantExpression("[1.0, 2.345]", new Double[]{1.0, 2.345});
validateConstantExpression("[1, 3]", new Long[]{1L, 3L});
validateConstantExpression("[\'hello\', \'world\']", new String[]{"hello", "world"});
validateConstantExpression("['hello', 'world']", new String[]{"hello", "world"});
}
@Test
public void testLiteralArraysHomogeneousOrNullElements()
{
validateConstantExpression("[1.0, null, 2.345]", new Double[]{1.0, null, 2.345});
validateConstantExpression("[null, 1, 3]", new Long[]{null, 1L, 3L});
validateConstantExpression("['hello', 'world', null]", new String[]{"hello", "world", null});
}
@Test
public void testLiteralArraysEmptyAndAllNullImplicitAreString()
{
validateConstantExpression("[]", new String[0]);
validateConstantExpression("[null, null, null]", new String[]{null, null, null});
}
@Test
public void testLiteralArraysImplicitTypedNumericMixed()
{
// implicit typed numeric arrays with mixed elements are doubles
validateConstantExpression("[1, null, 2000.0]", new Double[]{1.0, null, 2000.0});
validateConstantExpression("[1.0, null, 2000]", new Double[]{1.0, null, 2000.0});
}
@Test
public void testLiteralArraysExplicitTypedEmpties()
{
validateConstantExpression("<STRING>[]", new String[0]);
validateConstantExpression("<DOUBLE>[]", new Double[0]);
validateConstantExpression("<LONG>[]", new Long[0]);
}
@Test
public void testLiteralArraysExplicitAllNull()
{
validateConstantExpression("<DOUBLE>[null, null, null]", new Double[]{null, null, null});
validateConstantExpression("<LONG>[null, null, null]", new Long[]{null, null, null});
validateConstantExpression("<STRING>[null, null, null]", new String[]{null, null, null});
}
@Test
public void testLiteralArraysExplicitTypes()
{
validateConstantExpression("<DOUBLE>[1.0, null, 2000.0]", new Double[]{1.0, null, 2000.0});
validateConstantExpression("<LONG>[3, null, 4]", new Long[]{3L, null, 4L});
validateConstantExpression("<STRING>['foo', 'bar', 'baz']", new String[]{"foo", "bar", "baz"});
}
@Test
public void testLiteralArraysExplicitTypesMixedElements()
{
// explicit typed numeric arrays mixed numeric types should coerce to the correct explicit type
validateConstantExpression("<DOUBLE>[3, null, 4, 2.345]", new Double[]{3.0, null, 4.0, 2.345});
validateConstantExpression("<LONG>[1.0, null, 2000.0]", new Long[]{1L, null, 2000L});
// explicit typed string arrays should accept any literal and convert to string
validateConstantExpression("<STRING>['1', null, 2000, 1.1]", new String[]{"1", null, "2000", "1.1"});
}
@Test
public void testLiteralArrayImplicitStringParseException()
{
// implicit typed string array cannot handle literals thate are not null or string
expectedException.expect(RE.class);
expectedException.expectMessage("Failed to parse array: element 2000 is not a string");
validateConstantExpression("['1', null, 2000, 1.1]", new String[]{"1", null, "2000", "1.1"});
}
@Test
public void testLiteralArraysExplicitLongParseException()
{
// explicit typed long arrays only handle numeric types
expectedException.expect(RE.class);
expectedException.expectMessage("Failed to parse array element '2000' as a long");
validateConstantExpression("<LONG>[1, null, '2000']", new Long[]{1L, null, 2000L});
}
@Test
public void testLiteralArraysExplicitDoubleParseException()
{
// explicit typed double arrays only handle numeric types
expectedException.expect(RE.class);
expectedException.expectMessage("Failed to parse array element '2000.0' as a double");
validateConstantExpression("<DOUBLE>[1.0, null, '2000.0']", new Double[]{1.0, null, 2000.0});
}
@Test
@ -454,8 +545,17 @@ public class ParserTest extends InitializedNullHandlingTest
private void validateFlatten(String expression, String withoutFlatten, String withFlatten)
{
Assert.assertEquals(expression, withoutFlatten, Parser.parse(expression, ExprMacroTable.nil(), false).toString());
Assert.assertEquals(expression, withFlatten, Parser.parse(expression, ExprMacroTable.nil(), true).toString());
Expr notFlat = Parser.parse(expression, ExprMacroTable.nil(), false);
Expr flat = Parser.parse(expression, ExprMacroTable.nil(), true);
Assert.assertEquals(expression, withoutFlatten, notFlat.toString());
Assert.assertEquals(expression, withFlatten, flat.toString());
Expr notFlatRoundTrip = Parser.parse(notFlat.stringify(), ExprMacroTable.nil(), false);
Expr flatRoundTrip = Parser.parse(flat.stringify(), ExprMacroTable.nil(), true);
Assert.assertEquals(expression, withoutFlatten, notFlatRoundTrip.toString());
Assert.assertEquals(expression, withFlatten, flatRoundTrip.toString());
Assert.assertEquals(notFlat.stringify(), notFlatRoundTrip.stringify());
Assert.assertEquals(flat.stringify(), flatRoundTrip.stringify());
}
private void validateParser(String expression, String expected, List<String> identifiers)
@ -482,6 +582,14 @@ public class ParserTest extends InitializedNullHandlingTest
Assert.assertEquals(expression, identifiers, deets.getRequiredBindingsList());
Assert.assertEquals(expression, scalars, deets.getScalarVariables());
Assert.assertEquals(expression, arrays, deets.getArrayVariables());
final Expr parsedNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
final Expr roundTrip = Parser.parse(parsedNoFlatten.stringify(), ExprMacroTable.nil());
Assert.assertEquals(parsed.stringify(), roundTrip.stringify());
final Expr.BindingDetails roundTripDeets = roundTrip.analyzeInputs();
Assert.assertEquals(expression, identifiers, roundTripDeets.getRequiredBindingsList());
Assert.assertEquals(expression, scalars, roundTripDeets.getScalarVariables());
Assert.assertEquals(expression, arrays, roundTripDeets.getArrayVariables());
}
private void validateApplyUnapplied(
@ -497,23 +605,56 @@ public class ParserTest extends InitializedNullHandlingTest
final Expr transformed = Parser.applyUnappliedBindings(parsed, deets, identifiers);
Assert.assertEquals(expression, unapplied, parsed.toString());
Assert.assertEquals(applied, applied, transformed.toString());
final Expr parsedNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
final Expr parsedRoundTrip = Parser.parse(parsedNoFlatten.stringify(), ExprMacroTable.nil());
Expr.BindingDetails roundTripDeets = parsedRoundTrip.analyzeInputs();
Parser.validateExpr(parsedRoundTrip, roundTripDeets);
final Expr transformedRoundTrip = Parser.applyUnappliedBindings(parsedRoundTrip, roundTripDeets, identifiers);
Assert.assertEquals(expression, unapplied, parsedRoundTrip.toString());
Assert.assertEquals(applied, applied, transformedRoundTrip.toString());
Assert.assertEquals(parsed.stringify(), parsedRoundTrip.stringify());
Assert.assertEquals(transformed.stringify(), transformedRoundTrip.stringify());
}
private void validateConstantExpression(String expression, Object expected)
{
Expr parsed = Parser.parse(expression, ExprMacroTable.nil());
Assert.assertEquals(
expression,
expected,
Parser.parse(expression, ExprMacroTable.nil()).eval(Parser.withMap(ImmutableMap.of())).value()
parsed.eval(Parser.withMap(ImmutableMap.of())).value()
);
final Expr parsedNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
Expr parsedRoundTrip = Parser.parse(parsedNoFlatten.stringify(), ExprMacroTable.nil());
Assert.assertEquals(
expression,
expected,
parsedRoundTrip.eval(Parser.withMap(ImmutableMap.of())).value()
);
Assert.assertEquals(parsed.stringify(), parsedRoundTrip.stringify());
}
private void validateConstantExpression(String expression, Object[] expected)
{
Expr parsed = Parser.parse(expression, ExprMacroTable.nil());
Object evaluated = parsed.eval(Parser.withMap(ImmutableMap.of())).value();
Assert.assertArrayEquals(
expression,
expected,
(Object[]) Parser.parse(expression, ExprMacroTable.nil()).eval(Parser.withMap(ImmutableMap.of())).value()
(Object[]) evaluated
);
Assert.assertEquals(expected.getClass(), evaluated.getClass());
final Expr parsedNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
Expr roundTrip = Parser.parse(parsedNoFlatten.stringify(), ExprMacroTable.nil());
Assert.assertArrayEquals(
expression,
expected,
(Object[]) roundTrip.eval(Parser.withMap(ImmutableMap.of())).value()
);
Assert.assertEquals(parsed.stringify(), roundTrip.stringify());
}
}

28
docs/misc/math-expr.md
View File

@ -37,28 +37,15 @@ This expression language supports the following operators (listed in decreasing
|<, <=, >, >=, ==, !=|Binary Comparison|
|&&, &#124;&#124;|Binary Logical AND, OR|
Long, double, and string data types are supported. If a number contains a dot, it is interpreted as a double, otherwise
it is interpreted as a long. That means, always add a '.' to your number if you want it interpreted as a double value.
String literals should be quoted by single quotation marks.
Long, double, and string data types are supported. If a number contains a dot, it is interpreted as a double, otherwise it is interpreted as a long. That means, always add a '.' to your number if you want it interpreted as a double value. String literals should be quoted by single quotation marks.
Additionally, the expression language supports long, double, and string arrays. Array literals are created by wrapping
square brackets around a list of scalar literals values delimited by a comma or space character. All values in an array
literal must be the same type.
Additionally, the expression language supports long, double, and string arrays. Array literals are created by wrapping square brackets around a list of scalar literals values delimited by a comma or space character. All values in an array literal must be the same type, however null values are accepted. Typed empty arrays may be defined by prefixing with their type in angle brackets: `<STRING>[]`, `<DOUBLE>[]`, or `<LONG>[]`.
Expressions can contain variables. Variable names may contain letters, digits, '\_' and '$'. Variable names must not
begin with a digit. To escape other special characters, you can quote it with double quotation marks.
Expressions can contain variables. Variable names may contain letters, digits, '\_' and '$'. Variable names must not begin with a digit. To escape other special characters, you can quote it with double quotation marks.
For logical operators, a number is true if and only if it is positive (0 or negative value means false). For string
type, it's the evaluation result of 'Boolean.valueOf(string)'.
For logical operators, a number is true if and only if it is positive (0 or negative value means false). For string type, it's the evaluation result of 'Boolean.valueOf(string)'.
[Multi-value string dimensions](../querying/multi-value-dimensions.html) are supported and may be treated as either
scalar or array typed values. When treated as a scalar type, an expression will automatically be transformed to apply
the scalar operation across all values of the multi-valued type, to mimic Druid's native behavior. Values that result in
arrays will be coerced back into the native Druid string type for aggregation. Druid aggregations on multi-value string
dimensions on the individual values, _not_ the 'array', behaving similar to the `UNNEST` operator available in many SQL
dialects. However, by using the `array_to_string` function, aggregations may be done on a stringified version of the
complete array, allowing the complete row to be preserved. Using `string_to_array` in an expression post-aggregator,
allows transforming the stringified dimension back into the true native array type.
[Multi-value string dimensions](../querying/multi-value-dimensions.html) are supported and may be treated as either scalar or array typed values. When treated as a scalar type, an expression will automatically be transformed to apply the scalar operation across all values of the multi-valued type, to mimic Druid's native behavior. Values that result in arrays will be coerced back into the native Druid string type for aggregation. Druid aggregations on multi-value string dimensions on the individual values, _not_ the 'array', behaving similar to the `UNNEST` operator available in many SQL dialects. However, by using the `array_to_string` function, aggregations may be done on a stringified version of the complete array, allowing the complete row to be preserved. Using `string_to_array` in an expression post-aggregator, allows transforming the stringified dimension back into the true native array type.
The following built-in functions are available.
@ -196,10 +183,7 @@ See javadoc of java.lang.Math for detailed explanation for each function.
## IP address functions
For the IPv4 address functions, the `address` argument can either be an IPv4 dotted-decimal string
(e.g., "192.168.0.1") or an IP address represented as a long (e.g., 3232235521). The `subnet`
argument should be a string formatted as an IPv4 address subnet in CIDR notation (e.g.,
"192.168.0.0/16").
For the IPv4 address functions, the `address` argument can either be an IPv4 dotted-decimal string (e.g., "192.168.0.1") or an IP address represented as a long (e.g., 3232235521). The `subnet` argument should be a string formatted as an IPv4 address subnet in CIDR notation (e.g., "192.168.0.0/16").
| function | description |
| --- | --- |

2
extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/expressions/BloomFilterExprMacro.java
View File

@ -71,7 +71,7 @@ public class BloomFilterExprMacro implements ExprMacroTable.ExprMacro
{
private BloomExpr(Expr arg)
{
super(arg);
super(FN_NAME, arg);
}
@Nonnull

13
processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java
View File

@ -21,6 +21,7 @@ package org.apache.druid.query.expression;
import org.apache.commons.net.util.SubnetUtils;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExprMacroTable;
@ -52,13 +53,13 @@ import java.util.List;
*/
public class IPv4AddressMatchExprMacro implements ExprMacroTable.ExprMacro
{
public static final String NAME = "ipv4_match";
public static final String FN_NAME = "ipv4_match";
private static final int ARG_SUBNET = 1;
@Override
public String name()
{
return NAME;
return FN_NAME;
}
@Override
@ -77,7 +78,7 @@ public class IPv4AddressMatchExprMacro implements ExprMacroTable.ExprMacro
private IPv4AddressMatchExpr(Expr arg, SubnetUtils.SubnetInfo subnetInfo)
{
super(arg);
super(FN_NAME, arg);
this.subnetInfo = subnetInfo;
}
@ -116,6 +117,12 @@ public class IPv4AddressMatchExprMacro implements ExprMacroTable.ExprMacro
Expr newArg = arg.visit(shuttle);
return shuttle.visit(new IPv4AddressMatchExpr(newArg, subnetInfo));
}
@Override
public String stringify()
{
return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), args.get(ARG_SUBNET).stringify());
}
}
return new IPv4AddressMatchExpr(arg, subnetInfo);

6
processing/src/main/java/org/apache/druid/query/expression/IPv4AddressParseExprMacro.java
View File

@ -47,12 +47,12 @@ import java.util.List;
*/
public class IPv4AddressParseExprMacro implements ExprMacroTable.ExprMacro
{
public static final String NAME = "ipv4_parse";
public static final String FN_NAME = "ipv4_parse";
@Override
public String name()
{
return NAME;
return FN_NAME;
}
@Override
@ -68,7 +68,7 @@ public class IPv4AddressParseExprMacro implements ExprMacroTable.ExprMacro
{
private IPv4AddressParseExpr(Expr arg)
{
super(arg);
super(FN_NAME, arg);
}
@Nonnull

6
processing/src/main/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacro.java
View File

@ -46,12 +46,12 @@ import java.util.List;
*/
public class IPv4AddressStringifyExprMacro implements ExprMacroTable.ExprMacro
{
public static final String NAME = "ipv4_stringify";
public static final String FN_NAME = "ipv4_stringify";
@Override
public String name()
{
return NAME;
return FN_NAME;
}
@Override
@ -67,7 +67,7 @@ public class IPv4AddressStringifyExprMacro implements ExprMacroTable.ExprMacro
{
private IPv4AddressStringifyExpr(Expr arg)
{
super(arg);
super(FN_NAME, arg);
}
@Nonnull

22
processing/src/main/java/org/apache/druid/query/expression/LikeExprMacro.java
View File

@ -21,6 +21,7 @@ package org.apache.druid.query.expression;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExprMacroTable;
@ -32,10 +33,12 @@ import java.util.List;
public class LikeExprMacro implements ExprMacroTable.ExprMacro
{
private static final String FN_NAME = "like";
@Override
public String name()
{
return "like";
return FN_NAME;
}
@Override
@ -71,7 +74,7 @@ public class LikeExprMacro implements ExprMacroTable.ExprMacro
{
private LikeExtractExpr(Expr arg)
{
super(arg);
super(FN_NAME, arg);
}
@Nonnull
@ -87,6 +90,21 @@ public class LikeExprMacro implements ExprMacroTable.ExprMacro
Expr newArg = arg.visit(shuttle);
return shuttle.visit(new LikeExtractExpr(newArg));
}
@Override
public String stringify()
{
if (escapeExpr != null) {
return StringUtils.format(
"%s(%s, %s, %s)",
FN_NAME,
arg.stringify(),
patternExpr.stringify(),
escapeExpr.stringify()
);
}
return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), patternExpr.stringify());
}
}
return new LikeExtractExpr(arg);
}

12
processing/src/main/java/org/apache/druid/query/expression/LookupExprMacro.java
View File

@ -22,6 +22,7 @@ package org.apache.druid.query.expression;
import com.google.inject.Inject;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExprMacroTable;
@ -33,6 +34,7 @@ import java.util.List;
public class LookupExprMacro implements ExprMacroTable.ExprMacro
{
private static final String FN_NAME = "lookup";
private final LookupExtractorFactoryContainerProvider lookupExtractorFactoryContainerProvider;
@Inject
@ -44,7 +46,7 @@ public class LookupExprMacro implements ExprMacroTable.ExprMacro
@Override
public String name()
{
return "lookup";
return FN_NAME;
}
@Override
@ -75,7 +77,7 @@ public class LookupExprMacro implements ExprMacroTable.ExprMacro
{
private LookupExpr(Expr arg)
{
super(arg);
super(FN_NAME, arg);
}
@Nonnull
@ -91,6 +93,12 @@ public class LookupExprMacro implements ExprMacroTable.ExprMacro
Expr newArg = arg.visit(shuttle);
return shuttle.visit(new LookupExpr(newArg));
}
@Override
public String stringify()
{
return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), lookupExpr.stringify());
}
}
return new LookupExpr(arg);

22
processing/src/main/java/org/apache/druid/query/expression/RegexpExtractExprMacro.java
View File

@ -21,6 +21,7 @@ package org.apache.druid.query.expression;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExprMacroTable;
@ -32,10 +33,12 @@ import java.util.regex.Pattern;
public class RegexpExtractExprMacro implements ExprMacroTable.ExprMacro
{
private static final String FN_NAME = "regexp_extract";
@Override
public String name()
{
return "regexp_extract";
return FN_NAME;
}
@Override
@ -62,7 +65,7 @@ public class RegexpExtractExprMacro implements ExprMacroTable.ExprMacro
{
private RegexpExtractExpr(Expr arg)
{
super(arg);
super(FN_NAME, arg);
}
@Nonnull
@ -81,6 +84,21 @@ public class RegexpExtractExprMacro implements ExprMacroTable.ExprMacro
Expr newArg = arg.visit(shuttle);
return shuttle.visit(new RegexpExtractExpr(newArg));
}
@Override
public String stringify()
{
if (indexExpr != null) {
return StringUtils.format(
"%s(%s, %s, %s)",
FN_NAME,
arg.stringify(),
patternExpr.stringify(),
indexExpr.stringify()
);
}
return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), patternExpr.stringify());
}
}
return new RegexpExtractExpr(arg);
}

8
processing/src/main/java/org/apache/druid/query/expression/TimestampCeilExprMacro.java
View File

@ -34,10 +34,12 @@ import java.util.stream.Collectors;
public class TimestampCeilExprMacro implements ExprMacroTable.ExprMacro
{
private static final String FN_NAME = "timestamp_ceil";
@Override
public String name()
{
return "timestamp_ceil";
return FN_NAME;
}
@Override
@ -60,7 +62,7 @@ public class TimestampCeilExprMacro implements ExprMacroTable.ExprMacro
TimestampCeilExpr(final List<Expr> args)
{
super(args);
super(FN_NAME, args);
this.granularity = getGranularity(args, ExprUtils.nilBindings());
}
@ -103,7 +105,7 @@ public class TimestampCeilExprMacro implements ExprMacroTable.ExprMacro
{
TimestampCeilDynamicExpr(final List<Expr> args)
{
super(args);
super(FN_NAME, args);
}
@Nonnull

21
processing/src/main/java/org/apache/druid/query/expression/TimestampExtractExprMacro.java
View File

@ -34,6 +34,8 @@ import java.util.List;
public class TimestampExtractExprMacro implements ExprMacroTable.ExprMacro
{
private static final String FN_NAME = "timestamp_extract";
public enum Unit
{
EPOCH,
@ -59,7 +61,7 @@ public class TimestampExtractExprMacro implements ExprMacroTable.ExprMacro
@Override
public String name()
{
return "timestamp_extract";
return FN_NAME;
}
@Override
@ -93,7 +95,7 @@ public class TimestampExtractExprMacro implements ExprMacroTable.ExprMacro
{
private TimestampExtractExpr(Expr arg)
{
super(arg);
super(FN_NAME, arg);
}
@Nonnull
@ -159,6 +161,21 @@ public class TimestampExtractExprMacro implements ExprMacroTable.ExprMacro
Expr newArg = arg.visit(shuttle);
return shuttle.visit(new TimestampExtractExpr(newArg));
}
@Override
public String stringify()
{
if (args.size() > 2) {
return StringUtils.format(
"%s(%s, %s, %s)",
FN_NAME,
arg.stringify(),
args.get(1).stringify(),
args.get(2).stringify()
);
}
return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), args.get(1).stringify());
}
}
return new TimestampExtractExpr(arg);

8
processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java
View File

@ -32,10 +32,12 @@ import java.util.stream.Collectors;
public class TimestampFloorExprMacro implements ExprMacroTable.ExprMacro
{
private static final String FN_NAME = "timestamp_floor";
@Override
public String name()
{
return "timestamp_floor";
return FN_NAME;
}
@Override
@ -68,7 +70,7 @@ public class TimestampFloorExprMacro implements ExprMacroTable.ExprMacro
TimestampFloorExpr(final List<Expr> args)
{
super(args);
super(FN_NAME, args);
this.granularity = computeGranularity(args, ExprUtils.nilBindings());
}
@ -113,7 +115,7 @@ public class TimestampFloorExprMacro implements ExprMacroTable.ExprMacro
{
TimestampFloorDynamicExpr(final List<Expr> args)
{
super(args);
super(FN_NAME, args);
}
@Nonnull

25
processing/src/main/java/org/apache/druid/query/expression/TimestampFormatExprMacro.java
View File

@ -21,6 +21,7 @@ package org.apache.druid.query.expression;
import com.google.common.base.Preconditions;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExprMacroTable;
@ -34,10 +35,12 @@ import java.util.List;
public class TimestampFormatExprMacro implements ExprMacroTable.ExprMacro
{
private static final String FN_NAME = "timestamp_format";
@Override
public String name()
{
return "timestamp_format";
return FN_NAME;
}
@Override
@ -72,7 +75,7 @@ public class TimestampFormatExprMacro implements ExprMacroTable.ExprMacro
{
private TimestampFormatExpr(Expr arg)
{
super(arg);
super(FN_NAME, arg);
}
@Nonnull
@ -93,6 +96,24 @@ public class TimestampFormatExprMacro implements ExprMacroTable.ExprMacro
Expr newArg = arg.visit(shuttle);
return shuttle.visit(new TimestampFormatExpr(newArg));
}
@Override
public String stringify()
{
if (args.size() > 2) {
return StringUtils.format(
"%s(%s, %s, %s)",
FN_NAME,
arg.stringify(),
args.get(1).stringify(),
args.get(2).stringify()
);
}
if (args.size() > 1) {
return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), args.get(1).stringify());
}
return super.stringify();
}
}
return new TimestampFormatExpr(arg);

25
processing/src/main/java/org/apache/druid/query/expression/TimestampParseExprMacro.java
View File

@ -21,6 +21,7 @@ package org.apache.druid.query.expression;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExprMacroTable;
@ -36,10 +37,12 @@ import java.util.List;
public class TimestampParseExprMacro implements ExprMacroTable.ExprMacro
{
private static final String FN_NAME = "timestamp_parse";
@Override
public String name()
{
return "timestamp_parse";
return FN_NAME;
}
@Override
@ -68,7 +71,7 @@ public class TimestampParseExprMacro implements ExprMacroTable.ExprMacro
{
private TimestampParseExpr(Expr arg)
{
super(arg);
super(FN_NAME, arg);
}
@Nonnull
@ -96,6 +99,24 @@ public class TimestampParseExprMacro implements ExprMacroTable.ExprMacro
Expr newArg = arg.visit(shuttle);
return shuttle.visit(new TimestampParseExpr(newArg));
}
@Override
public String stringify()
{
if (args.size() > 2) {
return StringUtils.format(
"%s(%s, %s, %s)",
FN_NAME,
arg.stringify(),
args.get(1).stringify(),
args.get(2).stringify()
);
}
if (args.size() > 1) {
return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), args.get(1).stringify());
}
return super.stringify();
}
}
return new TimestampParseExpr(arg);

8
processing/src/main/java/org/apache/druid/query/expression/TimestampShiftExprMacro.java
View File

@ -34,10 +34,12 @@ import java.util.stream.Collectors;
public class TimestampShiftExprMacro implements ExprMacroTable.ExprMacro
{
private static final String FN_NAME = "timestamp_shift";
@Override
public String name()
{
return "timestamp_shift";
return FN_NAME;
}
@Override
@ -79,7 +81,7 @@ public class TimestampShiftExprMacro implements ExprMacroTable.ExprMacro
TimestampShiftExpr(final List<Expr> args)
{
super(args);
super(FN_NAME, args);
final PeriodGranularity granularity = getGranularity(args, ExprUtils.nilBindings());
period = granularity.getPeriod();
chronology = ISOChronology.getInstance(granularity.getTimeZone());
@ -105,7 +107,7 @@ public class TimestampShiftExprMacro implements ExprMacroTable.ExprMacro
{
TimestampShiftDynamicExpr(final List<Expr> args)
{
super(args);
super(FN_NAME, args);
}
@Nonnull

55
processing/src/main/java/org/apache/druid/query/expression/TrimExprMacro.java
View File

@ -21,6 +21,7 @@ package org.apache.druid.query.expression;
import com.google.common.collect.ImmutableSet;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExprMacroTable;
@ -35,19 +36,26 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
enum TrimMode
{
BOTH(true, true),
LEFT(true, false),
RIGHT(false, true);
BOTH("trim", true, true),
LEFT("ltrim", true, false),
RIGHT("rtrim", false, true);
private final String name;
private final boolean left;
private final boolean right;
TrimMode(final boolean left, final boolean right)
TrimMode(final String name, final boolean left, final boolean right)
{
this.name = name;
this.left = left;
this.right = right;
}
public String getFnName()
{
return name;
}
public boolean isLeft()
{
return left;
@ -60,18 +68,16 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
}
private final TrimMode mode;
private final String name;
public TrimExprMacro(final String name, final TrimMode mode)
public TrimExprMacro(final TrimMode mode)
{
this.name = name;
this.mode = mode;
}
@Override
public String name()
{
return name;
return mode.getFnName();
}
@Override
@ -82,13 +88,13 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
}
if (args.size() == 1) {
return new TrimStaticCharsExpr(mode, args.get(0), DEFAULT_CHARS);
return new TrimStaticCharsExpr(mode, args.get(0), DEFAULT_CHARS, null);
} else {
final Expr charsArg = args.get(1);
if (charsArg.isLiteral()) {
final String charsString = charsArg.eval(ExprUtils.nilBindings()).asString();
final char[] chars = charsString == null ? EMPTY_CHARS : charsString.toCharArray();
return new TrimStaticCharsExpr(mode, args.get(0), chars);
return new TrimStaticCharsExpr(mode, args.get(0), chars, charsArg);
} else {
return new TrimDynamicCharsExpr(mode, args.get(0), args.get(1));
}
@ -99,12 +105,14 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
{
private final TrimMode mode;
private final char[] chars;
private final Expr charsExpr;
public TrimStaticCharsExpr(final TrimMode mode, final Expr stringExpr, final char[] chars)
public TrimStaticCharsExpr(final TrimMode mode, final Expr stringExpr, final char[] chars, final Expr charsExpr)
{
super(stringExpr);
super(mode.getFnName(), stringExpr);
this.mode = mode;
this.chars = chars;
this.charsExpr = charsExpr;
}
@Nonnull
@ -153,7 +161,16 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
public Expr visit(Shuttle shuttle)
{
Expr newStringExpr = arg.visit(shuttle);
return shuttle.visit(new TrimStaticCharsExpr(mode, newStringExpr, chars));
return shuttle.visit(new TrimStaticCharsExpr(mode, newStringExpr, chars, charsExpr));
}
@Override
public String stringify()
{
if (charsExpr != null) {
return StringUtils.format("%s(%s, %s)", mode.getFnName(), arg.stringify(), charsExpr.stringify());
}
return super.stringify();
}
}
@ -219,6 +236,12 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
}
}
@Override
public String stringify()
{
return StringUtils.format("%s(%s, %s)", mode.getFnName(), stringExpr.stringify(), charsExpr.stringify());
}
@Override
public void visit(final Visitor visitor)
{
@ -270,7 +293,7 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
{
public BothTrimExprMacro()
{
super("trim", TrimMode.BOTH);
super(TrimMode.BOTH);
}
}
@ -278,7 +301,7 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
{
public LeftTrimExprMacro()
{
super("ltrim", TrimMode.LEFT);
super(TrimMode.LEFT);
}
}
@ -286,7 +309,7 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
{
public RightTrimExprMacro()
{
super("rtrim", TrimMode.RIGHT);
super(TrimMode.RIGHT);
}
}
}

2
processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacroTest.java
View File

@ -193,7 +193,7 @@ public class IPv4AddressMatchExprMacroTest extends MacroTestBase
{
NotLiteralExpr(Expr arg)
{
super(arg);
super("not", arg);
}
@Override

8
server/src/test/java/org/apache/druid/query/expression/ExprMacroTest.java
View File

@ -232,5 +232,13 @@ public class ExprMacroTest
{
final Expr expr = Parser.parse(expression, LookupEnabledTestExprMacroTable.INSTANCE);
Assert.assertEquals(expression, expectedResult, expr.eval(BINDINGS).value());
final Expr exprNotFlattened = Parser.parse(expression, LookupEnabledTestExprMacroTable.INSTANCE, false);
final Expr roundTripNotFlattened =
Parser.parse(exprNotFlattened.stringify(), LookupEnabledTestExprMacroTable.INSTANCE);
Assert.assertEquals(exprNotFlattened.stringify(), expectedResult, roundTripNotFlattened.eval(BINDINGS).value());
final Expr roundTrip = Parser.parse(expr.stringify(), LookupEnabledTestExprMacroTable.INSTANCE);
Assert.assertEquals(exprNotFlattened.stringify(), expectedResult, roundTrip.eval(BINDINGS).value());
}
}

4
sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/IPv4AddressMatchOperatorConversion.java
View File

@ -41,7 +41,7 @@ public class IPv4AddressMatchOperatorConversion extends DirectOperatorConversion
private static final SqlSingleOperandTypeChecker SUBNET_OPERAND = OperandTypes.family(SqlTypeFamily.STRING);
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder(StringUtils.toUpperCase(IPv4AddressMatchExprMacro.NAME))
.operatorBuilder(StringUtils.toUpperCase(IPv4AddressMatchExprMacro.FN_NAME))
.operandTypeChecker(OperandTypes.sequence("(expr,string)", ADDRESS_OPERAND, SUBNET_OPERAND))
.returnTypeInference(ReturnTypes.BOOLEAN_NULLABLE)
.functionCategory(SqlFunctionCategory.USER_DEFINED_FUNCTION)
@ -49,7 +49,7 @@ public class IPv4AddressMatchOperatorConversion extends DirectOperatorConversion
public IPv4AddressMatchOperatorConversion()
{
super(SQL_FUNCTION, IPv4AddressMatchExprMacro.NAME);
super(SQL_FUNCTION, IPv4AddressMatchExprMacro.FN_NAME);
}
@Override

4
sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/IPv4AddressParseOperatorConversion.java
View File

@ -33,7 +33,7 @@ import org.apache.druid.sql.calcite.expression.OperatorConversions;
public class IPv4AddressParseOperatorConversion extends DirectOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder(StringUtils.toUpperCase(IPv4AddressParseExprMacro.NAME))
.operatorBuilder(StringUtils.toUpperCase(IPv4AddressParseExprMacro.FN_NAME))
.operandTypeChecker(
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.STRING),
@ -45,7 +45,7 @@ public class IPv4AddressParseOperatorConversion extends DirectOperatorConversion
public IPv4AddressParseOperatorConversion()
{
super(SQL_FUNCTION, IPv4AddressParseExprMacro.NAME);
super(SQL_FUNCTION, IPv4AddressParseExprMacro.FN_NAME);
}
@Override

4
sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/IPv4AddressStringifyOperatorConversion.java
View File

@ -33,7 +33,7 @@ import org.apache.druid.sql.calcite.expression.OperatorConversions;
public class IPv4AddressStringifyOperatorConversion extends DirectOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder(StringUtils.toUpperCase(IPv4AddressStringifyExprMacro.NAME))
.operatorBuilder(StringUtils.toUpperCase(IPv4AddressStringifyExprMacro.FN_NAME))
.operandTypeChecker(
OperandTypes.or(
OperandTypes.family(SqlTypeFamily.INTEGER),
@ -45,7 +45,7 @@ public class IPv4AddressStringifyOperatorConversion extends DirectOperatorConver
public IPv4AddressStringifyOperatorConversion()
{
super(SQL_FUNCTION, IPv4AddressStringifyExprMacro.NAME);
super(SQL_FUNCTION, IPv4AddressStringifyExprMacro.FN_NAME);
}