string -> expression -> string -> expression (#9367)

* add Expr.stringify which produces parseable expression strings, parser support for null values in arrays, and parser support for empty numeric arrays

* oops, macros are expressions too

* style

* spotbugs

* qualified type arrays

* review stuffs

* simplify grammar

* more permissive array parsing

* reuse expr joiner

* fix it
This commit is contained in:
Clint Wylie 2020-02-21 15:43:02 -08:00 committed by GitHub
parent 05258dca37
commit 6d8dd5ec10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 750 additions and 137 deletions

View File

@ -15,32 +15,42 @@
grammar Expr; grammar Expr;
expr : 'null' # null expr : NULL # null
| ('-'|'!') expr # unaryOpExpr | ('-'|'!') expr # unaryOpExpr
|<assoc=right> expr '^' expr # powOpExpr |<assoc=right> expr '^' expr # powOpExpr
| expr ('*'|'/'|'%') expr # mulDivModuloExpr | expr ('*'|'/'|'%') expr # mulDivModuloExpr
| expr ('+'|'-') expr # addSubExpr | expr ('+'|'-') expr # addSubExpr
| expr ('<'|'<='|'>'|'>='|'=='|'!=') expr # logicalOpExpr | expr ('<'|'<='|'>'|'>='|'=='|'!=') expr # logicalOpExpr
| expr ('&&'|'||') expr # logicalAndOrExpr | expr ('&&'|'||') expr # logicalAndOrExpr
| '(' expr ')' # nestedExpr | '(' expr ')' # nestedExpr
| IDENTIFIER '(' lambda ',' fnArgs ')' # applyFunctionExpr | IDENTIFIER '(' lambda ',' fnArgs ')' # applyFunctionExpr
| IDENTIFIER '(' fnArgs? ')' # functionExpr | IDENTIFIER '(' fnArgs? ')' # functionExpr
| IDENTIFIER # identifierExpr | IDENTIFIER # identifierExpr
| DOUBLE # doubleExpr | DOUBLE # doubleExpr
| LONG # longExpr | LONG # longExpr
| STRING # string | STRING # string
| '[' DOUBLE (',' DOUBLE)* ']' # doubleArray | '[' (stringElement (',' stringElement)*)? ']' # stringArray
| '[' LONG (',' LONG)* ']' # longArray | '[' longElement (',' longElement)*']' # longArray
| '[' STRING (',' STRING)* ']' # stringArray | '<LONG>' '[' (numericElement (',' numericElement)*)? ']' # explicitLongArray
| '[]' # emptyArray | '<DOUBLE>'? '[' (numericElement (',' numericElement)*)? ']' # doubleArray
| '<STRING>' '[' (literalElement (',' literalElement)*)? ']' # explicitStringArray
; ;
lambda : (IDENTIFIER | '(' ')' | '(' IDENTIFIER (',' IDENTIFIER)* ')') '->' expr lambda : (IDENTIFIER | '(' ')' | '(' IDENTIFIER (',' IDENTIFIER)* ')') '->' expr
; ;
fnArgs : expr (',' expr)* # functionArgs fnArgs : expr (',' expr)* # functionArgs
; ;
stringElement : (STRING | NULL);
longElement : (LONG | NULL);
numericElement : (LONG | DOUBLE | NULL);
literalElement : (STRING | LONG | DOUBLE | NULL);
NULL : 'null';
IDENTIFIER : [_$a-zA-Z][_$a-zA-Z0-9]* | '"' (ESC | ~ [\"\\])* '"'; IDENTIFIER : [_$a-zA-Z][_$a-zA-Z0-9]* | '"' (ESC | ~ [\"\\])* '"';
LONG : [0-9]+ ; LONG : [0-9]+ ;
DOUBLE : [0-9]+ '.' [0-9]* ; DOUBLE : [0-9]+ '.' [0-9]* ;

View File

@ -118,6 +118,7 @@ public final class Numbers
} }
} }
/** /**
* Try parsing the given Number or String object val as long. * Try parsing the given Number or String object val as long.
* @param val * @param val
@ -167,6 +168,45 @@ public final class Numbers
} }
} }
/**
* Like {@link #tryParseDouble}, but does not produce a primitive and will explode if unable to produce a Double
* similar to {@link Double#parseDouble}
*/
@Nullable
public static Double parseDoubleObject(@Nullable String val)
{
if (val == null) {
return null;
}
Double d = Doubles.tryParse(val);
if (d != null) {
return d;
}
throw new NumberFormatException("Cannot parse string to double");
}
/**
* Like {@link #tryParseLong} but does not produce a primitive and will explode if unable to produce a Long
* similar to {@link Long#parseLong}
*/
@Nullable
public static Long parseLongObject(@Nullable String val)
{
if (val == null) {
return null;
}
Long lobj = Longs.tryParse(val);
if (lobj != null) {
return lobj;
}
// try as a double, for "ddd.dd" , Longs.tryParse(..) returns null
Double dobj = Doubles.tryParse((String) val);
if (dobj != null) {
return dobj.longValue();
}
throw new NumberFormatException("Cannot parse string to long");
}
public static int toIntExact(long value, String error) public static int toIntExact(long value, String error)
{ {
if ((int) value != value) { if ((int) value != value) {

View File

@ -19,12 +19,14 @@
package org.apache.druid.math.expr; package org.apache.druid.math.expr;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
import com.google.common.math.LongMath; import com.google.common.math.LongMath;
import com.google.common.primitives.Ints; import com.google.common.primitives.Ints;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.druid.common.config.NullHandling; import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.ISE;
@ -46,6 +48,8 @@ import java.util.stream.Collectors;
*/ */
public interface Expr public interface Expr
{ {
String NULL_LITERAL = "null";
Joiner ARG_JOINER = Joiner.on(", ");
/** /**
* Indicates expression is a constant whose literal value can be extracted by {@link Expr#getLiteralValue()}, * Indicates expression is a constant whose literal value can be extracted by {@link Expr#getLiteralValue()},
* making evaluating with arguments and bindings unecessary * making evaluating with arguments and bindings unecessary
@ -109,6 +113,12 @@ public interface Expr
*/ */
ExprEval eval(ObjectBinding bindings); ExprEval eval(ObjectBinding bindings);
/**
* Convert the {@link Expr} back into parseable string that when parsed with
* {@link Parser#parse(String, ExprMacroTable)} will produce an equivalent {@link Expr}.
*/
String stringify();
/** /**
* Programmatically inspect the {@link Expr} tree with a {@link Visitor}. Each {@link Expr} is responsible for * Programmatically inspect the {@link Expr} tree with a {@link Visitor}. Each {@link Expr} is responsible for
* ensuring the {@link Visitor} can visit all of its {@link Expr} children before visiting itself * ensuring the {@link Visitor} can visit all of its {@link Expr} children before visiting itself
@ -463,6 +473,12 @@ abstract class ConstantExpr implements Expr
{ {
return new BindingDetails(); return new BindingDetails();
} }
@Override
public String stringify()
{
return toString();
}
} }
abstract class NullNumericConstantExpr extends ConstantExpr abstract class NullNumericConstantExpr extends ConstantExpr
@ -476,7 +492,7 @@ abstract class NullNumericConstantExpr extends ConstantExpr
@Override @Override
public String toString() public String toString()
{ {
return "null"; return NULL_LITERAL;
} }
} }
@ -544,6 +560,15 @@ class LongArrayExpr extends ConstantExpr
{ {
return ExprEval.ofLongArray(value); return ExprEval.ofLongArray(value);
} }
@Override
public String stringify()
{
if (value.length == 0) {
return "<LONG>[]";
}
return StringUtils.format("<LONG>%s", toString());
}
} }
class StringExpr extends ConstantExpr class StringExpr extends ConstantExpr
@ -574,6 +599,13 @@ class StringExpr extends ConstantExpr
{ {
return ExprEval.of(value); return ExprEval.of(value);
} }
@Override
public String stringify()
{
// escape as javascript string since string literals are wrapped in single quotes
return value == null ? NULL_LITERAL : StringUtils.format("'%s'", StringEscapeUtils.escapeJavaScript(value));
}
} }
class StringArrayExpr extends ConstantExpr class StringArrayExpr extends ConstantExpr
@ -602,6 +634,27 @@ class StringArrayExpr extends ConstantExpr
{ {
return ExprEval.ofStringArray(value); return ExprEval.ofStringArray(value);
} }
@Override
public String stringify()
{
if (value.length == 0) {
return "<STRING>[]";
}
return StringUtils.format(
"<STRING>[%s]",
ARG_JOINER.join(
Arrays.stream(value)
.map(s -> s == null
? NULL_LITERAL
// escape as javascript string since string literals are wrapped in single quotes
: StringUtils.format("'%s'", StringEscapeUtils.escapeJavaScript(s))
)
.iterator()
)
);
}
} }
class DoubleExpr extends ConstantExpr class DoubleExpr extends ConstantExpr
@ -667,6 +720,15 @@ class DoubleArrayExpr extends ConstantExpr
{ {
return ExprEval.ofDoubleArray(value); return ExprEval.ofDoubleArray(value);
} }
@Override
public String stringify()
{
if (value.length == 0) {
return "<DOUBLE>[]";
}
return StringUtils.format("<DOUBLE>%s", toString());
}
} }
/** /**
@ -757,6 +819,13 @@ class IdentifierExpr implements Expr
return ExprEval.bestEffortOf(bindings.get(binding)); return ExprEval.bestEffortOf(bindings.get(binding));
} }
@Override
public String stringify()
{
// escape as java strings since identifiers are wrapped in double quotes
return StringUtils.format("\"%s\"", StringEscapeUtils.escapeJava(binding));
}
@Override @Override
public void visit(Visitor visitor) public void visit(Visitor visitor)
{ {
@ -823,6 +892,12 @@ class LambdaExpr implements Expr
return expr.eval(bindings); return expr.eval(bindings);
} }
@Override
public String stringify()
{
return StringUtils.format("(%s) -> %s", ARG_JOINER.join(getIdentifiers()), expr.stringify());
}
@Override @Override
public void visit(Visitor visitor) public void visit(Visitor visitor)
{ {
@ -879,6 +954,12 @@ class FunctionExpr implements Expr
return function.apply(args, bindings); return function.apply(args, bindings);
} }
@Override
public String stringify()
{
return StringUtils.format("%s(%s)", name, ARG_JOINER.join(args.stream().map(Expr::stringify).iterator()));
}
@Override @Override
public void visit(Visitor visitor) public void visit(Visitor visitor)
{ {
@ -965,6 +1046,17 @@ class ApplyFunctionExpr implements Expr
return function.apply(lambdaExpr, argsExpr, bindings); return function.apply(lambdaExpr, argsExpr, bindings);
} }
@Override
public String stringify()
{
return StringUtils.format(
"%s(%s, %s)",
name,
lambdaExpr.stringify(),
ARG_JOINER.join(argsExpr.stream().map(Expr::stringify).iterator())
);
}
@Override @Override
public void visit(Visitor visitor) public void visit(Visitor visitor)
{ {
@ -1059,6 +1151,12 @@ class UnaryMinusExpr extends UnaryExpr
throw new IAE("unsupported type " + ret.type()); throw new IAE("unsupported type " + ret.type());
} }
@Override
public String stringify()
{
return StringUtils.format("-%s", expr.stringify());
}
@Override @Override
public String toString() public String toString()
{ {
@ -1091,6 +1189,12 @@ class UnaryNotExpr extends UnaryExpr
return ExprEval.of(!ret.asBoolean(), retType); return ExprEval.of(!ret.asBoolean(), retType);
} }
@Override
public String stringify()
{
return StringUtils.format("!%s", expr.stringify());
}
@Override @Override
public String toString() public String toString()
{ {
@ -1144,6 +1248,12 @@ abstract class BinaryOpExprBase implements Expr
return StringUtils.format("(%s %s %s)", op, left, right); return StringUtils.format("(%s %s %s)", op, left, right);
} }
@Override
public String stringify()
{
return StringUtils.format("(%s %s %s)", left.stringify(), op, right.stringify());
}
protected abstract BinaryOpExprBase copy(Expr left, Expr right); protected abstract BinaryOpExprBase copy(Expr left, Expr right);
@Override @Override

View File

@ -23,11 +23,13 @@ import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.TerminalNode; import org.antlr.v4.runtime.tree.TerminalNode;
import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.lang.StringEscapeUtils;
import org.apache.druid.annotations.UsedInGeneratedCode; import org.apache.druid.annotations.UsedInGeneratedCode;
import org.apache.druid.java.util.common.Numbers;
import org.apache.druid.java.util.common.RE; import org.apache.druid.java.util.common.RE;
import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.antlr.ExprBaseListener; import org.apache.druid.math.expr.antlr.ExprBaseListener;
import org.apache.druid.math.expr.antlr.ExprParser; import org.apache.druid.math.expr.antlr.ExprParser;
import javax.annotation.Nullable;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
@ -77,7 +79,7 @@ public class ExprListenerImpl extends ExprBaseListener
nodes.put(ctx, new UnaryNotExpr((Expr) nodes.get(ctx.getChild(1)))); nodes.put(ctx, new UnaryNotExpr((Expr) nodes.get(ctx.getChild(1))));
break; break;
default: default:
throw new RuntimeException("Unrecognized unary operator " + ctx.getChild(0).getText()); throw new RE("Unrecognized unary operator %s", ctx.getChild(0).getText());
} }
} }
@ -97,6 +99,7 @@ public class ExprListenerImpl extends ExprBaseListener
); );
} }
@Override @Override
public void exitDoubleExpr(ExprParser.DoubleExprContext ctx) public void exitDoubleExpr(ExprParser.DoubleExprContext ctx)
{ {
@ -106,15 +109,6 @@ public class ExprListenerImpl extends ExprBaseListener
); );
} }
@Override
public void exitDoubleArray(ExprParser.DoubleArrayContext ctx)
{
Double[] values = new Double[ctx.DOUBLE().size()];
for (int i = 0; i < values.length; i++) {
values[i] = Double.parseDouble(ctx.DOUBLE(i).getText());
}
nodes.put(ctx, new DoubleArrayExpr(values));
}
@Override @Override
public void exitAddSubExpr(ExprParser.AddSubExprContext ctx) public void exitAddSubExpr(ExprParser.AddSubExprContext ctx)
@ -142,7 +136,7 @@ public class ExprListenerImpl extends ExprBaseListener
); );
break; break;
default: default:
throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText()); throw new RE("Unrecognized binary operator %s", ctx.getChild(1).getText());
} }
} }
@ -181,20 +175,10 @@ public class ExprListenerImpl extends ExprBaseListener
); );
break; break;
default: default:
throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText()); throw new RE("Unrecognized binary operator %s", ctx.getChild(1).getText());
} }
} }
@Override
public void exitLongArray(ExprParser.LongArrayContext ctx)
{
Long[] values = new Long[ctx.LONG().size()];
for (int i = 0; i < values.length; i++) {
values[i] = Long.parseLong(ctx.LONG(i).getText());
}
nodes.put(ctx, new LongArrayExpr(values));
}
@Override @Override
public void exitNestedExpr(ExprParser.NestedExprContext ctx) public void exitNestedExpr(ExprParser.NestedExprContext ctx)
{ {
@ -273,7 +257,7 @@ public class ExprListenerImpl extends ExprBaseListener
); );
break; break;
default: default:
throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText()); throw new RE("Unrecognized binary operator %s", ctx.getChild(1).getText());
} }
} }
@ -313,7 +297,7 @@ public class ExprListenerImpl extends ExprBaseListener
); );
break; break;
default: default:
throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText()); throw new RE("Unrecognized binary operator %s", ctx.getChild(1).getText());
} }
} }
@ -403,20 +387,92 @@ public class ExprListenerImpl extends ExprBaseListener
nodes.put(ctx, new StringExpr(null)); nodes.put(ctx, new StringExpr(null));
} }
@Override
public void exitDoubleArray(ExprParser.DoubleArrayContext ctx)
{
Double[] values = new Double[ctx.numericElement().size()];
for (int i = 0; i < values.length; i++) {
if (ctx.numericElement(i).NULL() != null) {
values[i] = null;
} else if (ctx.numericElement(i).LONG() != null) {
values[i] = Numbers.parseDoubleObject(ctx.numericElement(i).LONG().getText());
} else if (ctx.numericElement(i).DOUBLE() != null) {
values[i] = Numbers.parseDoubleObject(ctx.numericElement(i).DOUBLE().getText());
} else {
throw new RE("Failed to parse array element %s as a double", ctx.numericElement(i).getText());
}
}
nodes.put(ctx, new DoubleArrayExpr(values));
}
@Override
public void exitLongArray(ExprParser.LongArrayContext ctx)
{
Long[] values = new Long[ctx.longElement().size()];
for (int i = 0; i < values.length; i++) {
if (ctx.longElement(i).NULL() != null) {
values[i] = null;
} else if (ctx.longElement(i).LONG() != null) {
values[i] = Long.parseLong(ctx.longElement(i).LONG().getText());
} else {
throw new RE("Failed to parse array element %s as a long", ctx.longElement(i).getText());
}
}
nodes.put(ctx, new LongArrayExpr(values));
}
@Override
public void exitExplicitLongArray(ExprParser.ExplicitLongArrayContext ctx)
{
Long[] values = new Long[ctx.numericElement().size()];
for (int i = 0; i < values.length; i++) {
if (ctx.numericElement(i).NULL() != null) {
values[i] = null;
} else if (ctx.numericElement(i).LONG() != null) {
values[i] = Numbers.parseLongObject(ctx.numericElement(i).LONG().getText());
} else if (ctx.numericElement(i).DOUBLE() != null) {
values[i] = Numbers.parseLongObject(ctx.numericElement(i).DOUBLE().getText());
} else {
throw new RE("Failed to parse array element %s as a long", ctx.numericElement(i).getText());
}
}
nodes.put(ctx, new LongArrayExpr(values));
}
@Override @Override
public void exitStringArray(ExprParser.StringArrayContext ctx) public void exitStringArray(ExprParser.StringArrayContext ctx)
{ {
String[] values = new String[ctx.STRING().size()]; String[] values = new String[ctx.stringElement().size()];
for (int i = 0; i < values.length; i++) { for (int i = 0; i < values.length; i++) {
values[i] = escapeStringLiteral(ctx.STRING(i).getText()); if (ctx.stringElement(i).NULL() != null) {
values[i] = null;
} else if (ctx.stringElement(i).STRING() != null) {
values[i] = escapeStringLiteral(ctx.stringElement(i).STRING().getText());
} else {
throw new RE("Failed to parse array: element %s is not a string", ctx.stringElement(i).getText());
}
} }
nodes.put(ctx, new StringArrayExpr(values)); nodes.put(ctx, new StringArrayExpr(values));
} }
@Override @Override
public void exitEmptyArray(ExprParser.EmptyArrayContext ctx) public void exitExplicitStringArray(ExprParser.ExplicitStringArrayContext ctx)
{ {
nodes.put(ctx, new StringArrayExpr(new String[0])); String[] values = new String[ctx.literalElement().size()];
for (int i = 0; i < values.length; i++) {
if (ctx.literalElement(i).NULL() != null) {
values[i] = null;
} else if (ctx.literalElement(i).STRING() != null) {
values[i] = escapeStringLiteral(ctx.literalElement(i).STRING().getText());
} else if (ctx.literalElement(i).DOUBLE() != null) {
values[i] = ctx.literalElement(i).DOUBLE().getText();
} else if (ctx.literalElement(i).LONG() != null) {
values[i] = ctx.literalElement(i).LONG().getText();
} else {
throw new RE("Failed to parse array element %s as a string", ctx.literalElement(i).getText());
}
}
nodes.put(ctx, new StringArrayExpr(values));
} }
/** /**
@ -455,8 +511,12 @@ public class ExprListenerImpl extends ExprBaseListener
/** /**
* Remove single quote from a string literal, returning unquoted string value * Remove single quote from a string literal, returning unquoted string value
*/ */
@Nullable
private static String escapeStringLiteral(String text) private static String escapeStringLiteral(String text)
{ {
if (text.equalsIgnoreCase(Expr.NULL_LITERAL)) {
return null;
}
String unquoted = text.substring(1, text.length() - 1); String unquoted = text.substring(1, text.length() - 1);
return unquoted.indexOf('\\') >= 0 ? StringEscapeUtils.unescapeJava(unquoted) : unquoted; return unquoted.indexOf('\\') >= 0 ? StringEscapeUtils.unescapeJava(unquoted) : unquoted;
} }

View File

@ -97,13 +97,15 @@ public class ExprMacroTable
*/ */
public abstract static class BaseScalarUnivariateMacroFunctionExpr implements Expr public abstract static class BaseScalarUnivariateMacroFunctionExpr implements Expr
{ {
protected final String name;
protected final Expr arg; protected final Expr arg;
// Use Supplier to memoize values as ExpressionSelectors#makeExprEvalSelector() can make repeated calls for them // Use Supplier to memoize values as ExpressionSelectors#makeExprEvalSelector() can make repeated calls for them
private final Supplier<BindingDetails> analyzeInputsSupplier; private final Supplier<BindingDetails> analyzeInputsSupplier;
public BaseScalarUnivariateMacroFunctionExpr(Expr arg) public BaseScalarUnivariateMacroFunctionExpr(String name, Expr arg)
{ {
this.name = name;
this.arg = arg; this.arg = arg;
analyzeInputsSupplier = Suppliers.memoize(this::supplyAnalyzeInputs); analyzeInputsSupplier = Suppliers.memoize(this::supplyAnalyzeInputs);
} }
@ -121,6 +123,12 @@ public class ExprMacroTable
return analyzeInputsSupplier.get(); return analyzeInputsSupplier.get();
} }
@Override
public String stringify()
{
return StringUtils.format("%s(%s)", name, arg.stringify());
}
private BindingDetails supplyAnalyzeInputs() private BindingDetails supplyAnalyzeInputs()
{ {
return arg.analyzeInputs().withScalarArguments(ImmutableSet.of(arg)); return arg.analyzeInputs().withScalarArguments(ImmutableSet.of(arg));
@ -132,17 +140,29 @@ public class ExprMacroTable
*/ */
public abstract static class BaseScalarMacroFunctionExpr implements Expr public abstract static class BaseScalarMacroFunctionExpr implements Expr
{ {
protected final String name;
protected final List<Expr> args; protected final List<Expr> args;
// Use Supplier to memoize values as ExpressionSelectors#makeExprEvalSelector() can make repeated calls for them // Use Supplier to memoize values as ExpressionSelectors#makeExprEvalSelector() can make repeated calls for them
private final Supplier<BindingDetails> analyzeInputsSupplier; private final Supplier<BindingDetails> analyzeInputsSupplier;
public BaseScalarMacroFunctionExpr(final List<Expr> args) public BaseScalarMacroFunctionExpr(String name, final List<Expr> args)
{ {
this.name = name;
this.args = args; this.args = args;
analyzeInputsSupplier = Suppliers.memoize(this::supplyAnalyzeInputs); analyzeInputsSupplier = Suppliers.memoize(this::supplyAnalyzeInputs);
} }
@Override
public String stringify()
{
return StringUtils.format(
"%s(%s)",
name,
Expr.ARG_JOINER.join(args.stream().map(Expr::stringify).iterator())
);
}
@Override @Override
public void visit(final Visitor visitor) public void visit(final Visitor visitor)
{ {

View File

@ -108,7 +108,7 @@ public class Parser
} }
@VisibleForTesting @VisibleForTesting
static Expr parse(String in, ExprMacroTable macroTable, boolean withFlatten) public static Expr parse(String in, ExprMacroTable macroTable, boolean withFlatten)
{ {
ExprLexer lexer = new ExprLexer(new ANTLRInputStream(in)); ExprLexer lexer = new ExprLexer(new ANTLRInputStream(in));
CommonTokenStream tokens = new CommonTokenStream(lexer); CommonTokenStream tokens = new CommonTokenStream(lexer);
@ -118,7 +118,11 @@ public class Parser
ParseTreeWalker walker = new ParseTreeWalker(); ParseTreeWalker walker = new ParseTreeWalker();
ExprListenerImpl listener = new ExprListenerImpl(parseTree, macroTable); ExprListenerImpl listener = new ExprListenerImpl(parseTree, macroTable);
walker.walk(listener, parseTree); walker.walk(listener, parseTree);
return withFlatten ? flatten(listener.getAST()) : listener.getAST(); Expr parsed = listener.getAST();
if (parsed == null) {
throw new RE("Failed to parse expression: %s", in);
}
return withFlatten ? flatten(parsed) : parsed;
} }
/** /**

View File

@ -127,4 +127,36 @@ public class NumbersTest
expectedException.expectMessage(CoreMatchers.startsWith("Unknown type")); expectedException.expectMessage(CoreMatchers.startsWith("Unknown type"));
Numbers.parseBoolean(new Object()); Numbers.parseBoolean(new Object());
} }
@Test
public void testParseLongObject()
{
Assert.assertEquals(null, Numbers.parseLongObject(null));
Assert.assertEquals((Long) 1L, Numbers.parseLongObject("1"));
Assert.assertEquals((Long) 32L, Numbers.parseLongObject("32.1243"));
}
@Test
public void testParseLongObjectUnparseable()
{
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("Cannot parse string to long");
Assert.assertEquals((Long) 1337L, Numbers.parseLongObject("'1'"));
}
@Test
public void testParseDoubleObject()
{
Assert.assertEquals(null, Numbers.parseLongObject(null));
Assert.assertEquals((Double) 1.0, Numbers.parseDoubleObject("1"));
Assert.assertEquals((Double) 32.1243, Numbers.parseDoubleObject("32.1243"));
}
@Test
public void testParseDoubleObjectUnparseable()
{
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("Cannot parse string to double");
Assert.assertEquals((Double) 300.0, Numbers.parseDoubleObject("'1.1'"));
}
} }

View File

@ -102,7 +102,7 @@ public class ApplyFunctionTest extends InitializedNullHandlingTest
assertExpr("fold((b, acc) -> b * acc, map((b) -> b * 2, filter(b -> b > 3, b)), 1)", 80L); assertExpr("fold((b, acc) -> b * acc, map((b) -> b * 2, filter(b -> b > 3, b)), 1)", 80L);
assertExpr("fold((a, acc) -> concat(a, acc), a, '')", "foobarbazbarfoo"); assertExpr("fold((a, acc) -> concat(a, acc), a, '')", "foobarbazbarfoo");
assertExpr("fold((a, acc) -> array_append(acc, a), a, [])", new String[]{"foo", "bar", "baz", "foobar"}); assertExpr("fold((a, acc) -> array_append(acc, a), a, [])", new String[]{"foo", "bar", "baz", "foobar"});
assertExpr("fold((a, acc) -> array_append(acc, a), b, cast([], 'LONG_ARRAY'))", new Long[]{1L, 2L, 3L, 4L, 5L}); assertExpr("fold((a, acc) -> array_append(acc, a), b, <LONG>[])", new Long[]{1L, 2L, 3L, 4L, 5L});
} }
@Test @Test
@ -161,6 +161,16 @@ public class ApplyFunctionTest extends InitializedNullHandlingTest
{ {
final Expr expr = Parser.parse(expression, ExprMacroTable.nil()); final Expr expr = Parser.parse(expression, ExprMacroTable.nil());
Assert.assertEquals(expression, expectedResult, expr.eval(bindings).value()); Assert.assertEquals(expression, expectedResult, expr.eval(bindings).value());
final Expr exprNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
final Expr roundTrip = Parser.parse(exprNoFlatten.stringify(), ExprMacroTable.nil());
Assert.assertEquals(expr.stringify(), expectedResult, roundTrip.eval(bindings).value());
final Expr roundTripFlatten = Parser.parse(expr.stringify(), ExprMacroTable.nil());
Assert.assertEquals(expr.stringify(), expectedResult, roundTripFlatten.eval(bindings).value());
Assert.assertEquals(expr.stringify(), roundTrip.stringify());
Assert.assertEquals(expr.stringify(), roundTripFlatten.stringify());
} }
private void assertExpr(final String expression, final Object[] expectedResult) private void assertExpr(final String expression, final Object[] expectedResult)
@ -170,6 +180,22 @@ public class ApplyFunctionTest extends InitializedNullHandlingTest
if (expectedResult.length != 0 || result == null || result.length != 0) { if (expectedResult.length != 0 || result == null || result.length != 0) {
Assert.assertArrayEquals(expression, expectedResult, result); Assert.assertArrayEquals(expression, expectedResult, result);
} }
final Expr exprNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
final Expr roundTrip = Parser.parse(exprNoFlatten.stringify(), ExprMacroTable.nil());
final Object[] resultRoundTrip = roundTrip.eval(bindings).asArray();
if (expectedResult.length != 0 || resultRoundTrip == null || resultRoundTrip.length != 0) {
Assert.assertArrayEquals(expr.stringify(), expectedResult, resultRoundTrip);
}
final Expr roundTripFlatten = Parser.parse(expr.stringify(), ExprMacroTable.nil());
final Object[] resultRoundTripFlatten = roundTripFlatten.eval(bindings).asArray();
if (expectedResult.length != 0 || resultRoundTripFlatten == null || resultRoundTripFlatten.length != 0) {
Assert.assertArrayEquals(expr.stringify(), expectedResult, resultRoundTripFlatten);
}
Assert.assertEquals(expr.stringify(), roundTrip.stringify());
Assert.assertEquals(expr.stringify(), roundTripFlatten.stringify());
} }
private void assertExpr(final String expression, final Double[] expectedResult) private void assertExpr(final String expression, final Double[] expectedResult)
@ -180,5 +206,23 @@ public class ApplyFunctionTest extends InitializedNullHandlingTest
for (int i = 0; i < result.length; i++) { for (int i = 0; i < result.length; i++) {
Assert.assertEquals(expression, expectedResult[i], result[i], 0.00001); // something is lame somewhere.. Assert.assertEquals(expression, expectedResult[i], result[i], 0.00001); // something is lame somewhere..
} }
final Expr exprNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
final Expr roundTrip = Parser.parse(exprNoFlatten.stringify(), ExprMacroTable.nil());
Double[] resultRoundTrip = (Double[]) roundTrip.eval(bindings).value();
Assert.assertEquals(expectedResult.length, resultRoundTrip.length);
for (int i = 0; i < resultRoundTrip.length; i++) {
Assert.assertEquals(expression, expectedResult[i], resultRoundTrip[i], 0.00001);
}
final Expr roundTripFlatten = Parser.parse(expr.stringify(), ExprMacroTable.nil());
Double[] resultRoundTripFlatten = (Double[]) roundTripFlatten.eval(bindings).value();
Assert.assertEquals(expectedResult.length, resultRoundTripFlatten.length);
for (int i = 0; i < resultRoundTripFlatten.length; i++) {
Assert.assertEquals(expression, expectedResult[i], resultRoundTripFlatten[i], 0.00001);
}
Assert.assertEquals(expr.stringify(), roundTrip.stringify());
Assert.assertEquals(expr.stringify(), roundTripFlatten.stringify());
} }
} }

View File

@ -231,7 +231,7 @@ public class FunctionTest extends InitializedNullHandlingTest
assertExpr("array_append([1, 2, 3], 4)", new Long[]{1L, 2L, 3L, 4L}); assertExpr("array_append([1, 2, 3], 4)", new Long[]{1L, 2L, 3L, 4L});
assertExpr("array_append([1, 2, 3], 'bar')", new Long[]{1L, 2L, 3L, null}); assertExpr("array_append([1, 2, 3], 'bar')", new Long[]{1L, 2L, 3L, null});
assertExpr("array_append([], 1)", new String[]{"1"}); assertExpr("array_append([], 1)", new String[]{"1"});
assertExpr("array_append(cast([], 'LONG_ARRAY'), 1)", new Long[]{1L}); assertExpr("array_append(<LONG>[], 1)", new Long[]{1L});
} }
@Test @Test
@ -287,18 +287,39 @@ public class FunctionTest extends InitializedNullHandlingTest
assertExpr("array_prepend(4, [1, 2, 3])", new Long[]{4L, 1L, 2L, 3L}); assertExpr("array_prepend(4, [1, 2, 3])", new Long[]{4L, 1L, 2L, 3L});
assertExpr("array_prepend('bar', [1, 2, 3])", new Long[]{null, 1L, 2L, 3L}); assertExpr("array_prepend('bar', [1, 2, 3])", new Long[]{null, 1L, 2L, 3L});
assertExpr("array_prepend(1, [])", new String[]{"1"}); assertExpr("array_prepend(1, [])", new String[]{"1"});
assertExpr("array_prepend(1, cast([], 'LONG_ARRAY'))", new Long[]{1L}); assertExpr("array_prepend(1, <LONG>[])", new Long[]{1L});
assertExpr("array_prepend(1, <DOUBLE>[])", new Double[]{1.0});
} }
private void assertExpr(final String expression, final Object expectedResult) private void assertExpr(final String expression, final Object expectedResult)
{ {
final Expr expr = Parser.parse(expression, ExprMacroTable.nil()); final Expr expr = Parser.parse(expression, ExprMacroTable.nil());
Assert.assertEquals(expression, expectedResult, expr.eval(bindings).value()); Assert.assertEquals(expression, expectedResult, expr.eval(bindings).value());
final Expr exprNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
final Expr roundTrip = Parser.parse(exprNoFlatten.stringify(), ExprMacroTable.nil());
Assert.assertEquals(expr.stringify(), expectedResult, roundTrip.eval(bindings).value());
final Expr roundTripFlatten = Parser.parse(expr.stringify(), ExprMacroTable.nil());
Assert.assertEquals(expr.stringify(), expectedResult, roundTripFlatten.eval(bindings).value());
Assert.assertEquals(expr.stringify(), roundTrip.stringify());
Assert.assertEquals(expr.stringify(), roundTripFlatten.stringify());
} }
private void assertExpr(final String expression, final Object[] expectedResult) private void assertExpr(final String expression, final Object[] expectedResult)
{ {
final Expr expr = Parser.parse(expression, ExprMacroTable.nil()); final Expr expr = Parser.parse(expression, ExprMacroTable.nil());
Assert.assertArrayEquals(expression, expectedResult, expr.eval(bindings).asArray()); Assert.assertArrayEquals(expression, expectedResult, expr.eval(bindings).asArray());
final Expr exprNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
final Expr roundTrip = Parser.parse(exprNoFlatten.stringify(), ExprMacroTable.nil());
Assert.assertArrayEquals(expression, expectedResult, roundTrip.eval(bindings).asArray());
final Expr roundTripFlatten = Parser.parse(expr.stringify(), ExprMacroTable.nil());
Assert.assertArrayEquals(expression, expectedResult, roundTripFlatten.eval(bindings).asArray());
Assert.assertEquals(expr.stringify(), roundTrip.stringify());
Assert.assertEquals(expr.stringify(), roundTripFlatten.stringify());
} }
} }

View File

@ -22,9 +22,12 @@ package org.apache.druid.math.expr;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import org.apache.druid.java.util.common.RE;
import org.apache.druid.testing.InitializedNullHandlingTest; import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.rules.ExpectedException;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
@ -35,6 +38,9 @@ import java.util.Set;
*/ */
public class ParserTest extends InitializedNullHandlingTest public class ParserTest extends InitializedNullHandlingTest
{ {
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Test @Test
public void testSimple() public void testSimple()
{ {
@ -186,11 +192,96 @@ public class ParserTest extends InitializedNullHandlingTest
} }
@Test @Test
public void testLiteralArrays() public void testLiteralArraysHomogeneousElements()
{ {
validateConstantExpression("[1.0, 2.345]", new Double[]{1.0, 2.345}); validateConstantExpression("[1.0, 2.345]", new Double[]{1.0, 2.345});
validateConstantExpression("[1, 3]", new Long[]{1L, 3L}); validateConstantExpression("[1, 3]", new Long[]{1L, 3L});
validateConstantExpression("[\'hello\', \'world\']", new String[]{"hello", "world"}); validateConstantExpression("['hello', 'world']", new String[]{"hello", "world"});
}
@Test
public void testLiteralArraysHomogeneousOrNullElements()
{
validateConstantExpression("[1.0, null, 2.345]", new Double[]{1.0, null, 2.345});
validateConstantExpression("[null, 1, 3]", new Long[]{null, 1L, 3L});
validateConstantExpression("['hello', 'world', null]", new String[]{"hello", "world", null});
}
@Test
public void testLiteralArraysEmptyAndAllNullImplicitAreString()
{
validateConstantExpression("[]", new String[0]);
validateConstantExpression("[null, null, null]", new String[]{null, null, null});
}
@Test
public void testLiteralArraysImplicitTypedNumericMixed()
{
// implicit typed numeric arrays with mixed elements are doubles
validateConstantExpression("[1, null, 2000.0]", new Double[]{1.0, null, 2000.0});
validateConstantExpression("[1.0, null, 2000]", new Double[]{1.0, null, 2000.0});
}
@Test
public void testLiteralArraysExplicitTypedEmpties()
{
validateConstantExpression("<STRING>[]", new String[0]);
validateConstantExpression("<DOUBLE>[]", new Double[0]);
validateConstantExpression("<LONG>[]", new Long[0]);
}
@Test
public void testLiteralArraysExplicitAllNull()
{
validateConstantExpression("<DOUBLE>[null, null, null]", new Double[]{null, null, null});
validateConstantExpression("<LONG>[null, null, null]", new Long[]{null, null, null});
validateConstantExpression("<STRING>[null, null, null]", new String[]{null, null, null});
}
@Test
public void testLiteralArraysExplicitTypes()
{
validateConstantExpression("<DOUBLE>[1.0, null, 2000.0]", new Double[]{1.0, null, 2000.0});
validateConstantExpression("<LONG>[3, null, 4]", new Long[]{3L, null, 4L});
validateConstantExpression("<STRING>['foo', 'bar', 'baz']", new String[]{"foo", "bar", "baz"});
}
@Test
public void testLiteralArraysExplicitTypesMixedElements()
{
// explicit typed numeric arrays mixed numeric types should coerce to the correct explicit type
validateConstantExpression("<DOUBLE>[3, null, 4, 2.345]", new Double[]{3.0, null, 4.0, 2.345});
validateConstantExpression("<LONG>[1.0, null, 2000.0]", new Long[]{1L, null, 2000L});
// explicit typed string arrays should accept any literal and convert to string
validateConstantExpression("<STRING>['1', null, 2000, 1.1]", new String[]{"1", null, "2000", "1.1"});
}
@Test
public void testLiteralArrayImplicitStringParseException()
{
// implicit typed string array cannot handle literals thate are not null or string
expectedException.expect(RE.class);
expectedException.expectMessage("Failed to parse array: element 2000 is not a string");
validateConstantExpression("['1', null, 2000, 1.1]", new String[]{"1", null, "2000", "1.1"});
}
@Test
public void testLiteralArraysExplicitLongParseException()
{
// explicit typed long arrays only handle numeric types
expectedException.expect(RE.class);
expectedException.expectMessage("Failed to parse array element '2000' as a long");
validateConstantExpression("<LONG>[1, null, '2000']", new Long[]{1L, null, 2000L});
}
@Test
public void testLiteralArraysExplicitDoubleParseException()
{
// explicit typed double arrays only handle numeric types
expectedException.expect(RE.class);
expectedException.expectMessage("Failed to parse array element '2000.0' as a double");
validateConstantExpression("<DOUBLE>[1.0, null, '2000.0']", new Double[]{1.0, null, 2000.0});
} }
@Test @Test
@ -454,8 +545,17 @@ public class ParserTest extends InitializedNullHandlingTest
private void validateFlatten(String expression, String withoutFlatten, String withFlatten) private void validateFlatten(String expression, String withoutFlatten, String withFlatten)
{ {
Assert.assertEquals(expression, withoutFlatten, Parser.parse(expression, ExprMacroTable.nil(), false).toString()); Expr notFlat = Parser.parse(expression, ExprMacroTable.nil(), false);
Assert.assertEquals(expression, withFlatten, Parser.parse(expression, ExprMacroTable.nil(), true).toString()); Expr flat = Parser.parse(expression, ExprMacroTable.nil(), true);
Assert.assertEquals(expression, withoutFlatten, notFlat.toString());
Assert.assertEquals(expression, withFlatten, flat.toString());
Expr notFlatRoundTrip = Parser.parse(notFlat.stringify(), ExprMacroTable.nil(), false);
Expr flatRoundTrip = Parser.parse(flat.stringify(), ExprMacroTable.nil(), true);
Assert.assertEquals(expression, withoutFlatten, notFlatRoundTrip.toString());
Assert.assertEquals(expression, withFlatten, flatRoundTrip.toString());
Assert.assertEquals(notFlat.stringify(), notFlatRoundTrip.stringify());
Assert.assertEquals(flat.stringify(), flatRoundTrip.stringify());
} }
private void validateParser(String expression, String expected, List<String> identifiers) private void validateParser(String expression, String expected, List<String> identifiers)
@ -482,6 +582,14 @@ public class ParserTest extends InitializedNullHandlingTest
Assert.assertEquals(expression, identifiers, deets.getRequiredBindingsList()); Assert.assertEquals(expression, identifiers, deets.getRequiredBindingsList());
Assert.assertEquals(expression, scalars, deets.getScalarVariables()); Assert.assertEquals(expression, scalars, deets.getScalarVariables());
Assert.assertEquals(expression, arrays, deets.getArrayVariables()); Assert.assertEquals(expression, arrays, deets.getArrayVariables());
final Expr parsedNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
final Expr roundTrip = Parser.parse(parsedNoFlatten.stringify(), ExprMacroTable.nil());
Assert.assertEquals(parsed.stringify(), roundTrip.stringify());
final Expr.BindingDetails roundTripDeets = roundTrip.analyzeInputs();
Assert.assertEquals(expression, identifiers, roundTripDeets.getRequiredBindingsList());
Assert.assertEquals(expression, scalars, roundTripDeets.getScalarVariables());
Assert.assertEquals(expression, arrays, roundTripDeets.getArrayVariables());
} }
private void validateApplyUnapplied( private void validateApplyUnapplied(
@ -497,23 +605,56 @@ public class ParserTest extends InitializedNullHandlingTest
final Expr transformed = Parser.applyUnappliedBindings(parsed, deets, identifiers); final Expr transformed = Parser.applyUnappliedBindings(parsed, deets, identifiers);
Assert.assertEquals(expression, unapplied, parsed.toString()); Assert.assertEquals(expression, unapplied, parsed.toString());
Assert.assertEquals(applied, applied, transformed.toString()); Assert.assertEquals(applied, applied, transformed.toString());
final Expr parsedNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
final Expr parsedRoundTrip = Parser.parse(parsedNoFlatten.stringify(), ExprMacroTable.nil());
Expr.BindingDetails roundTripDeets = parsedRoundTrip.analyzeInputs();
Parser.validateExpr(parsedRoundTrip, roundTripDeets);
final Expr transformedRoundTrip = Parser.applyUnappliedBindings(parsedRoundTrip, roundTripDeets, identifiers);
Assert.assertEquals(expression, unapplied, parsedRoundTrip.toString());
Assert.assertEquals(applied, applied, transformedRoundTrip.toString());
Assert.assertEquals(parsed.stringify(), parsedRoundTrip.stringify());
Assert.assertEquals(transformed.stringify(), transformedRoundTrip.stringify());
} }
private void validateConstantExpression(String expression, Object expected) private void validateConstantExpression(String expression, Object expected)
{ {
Expr parsed = Parser.parse(expression, ExprMacroTable.nil());
Assert.assertEquals( Assert.assertEquals(
expression, expression,
expected, expected,
Parser.parse(expression, ExprMacroTable.nil()).eval(Parser.withMap(ImmutableMap.of())).value() parsed.eval(Parser.withMap(ImmutableMap.of())).value()
); );
final Expr parsedNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
Expr parsedRoundTrip = Parser.parse(parsedNoFlatten.stringify(), ExprMacroTable.nil());
Assert.assertEquals(
expression,
expected,
parsedRoundTrip.eval(Parser.withMap(ImmutableMap.of())).value()
);
Assert.assertEquals(parsed.stringify(), parsedRoundTrip.stringify());
} }
private void validateConstantExpression(String expression, Object[] expected) private void validateConstantExpression(String expression, Object[] expected)
{ {
Expr parsed = Parser.parse(expression, ExprMacroTable.nil());
Object evaluated = parsed.eval(Parser.withMap(ImmutableMap.of())).value();
Assert.assertArrayEquals( Assert.assertArrayEquals(
expression, expression,
expected, expected,
(Object[]) Parser.parse(expression, ExprMacroTable.nil()).eval(Parser.withMap(ImmutableMap.of())).value() (Object[]) evaluated
); );
Assert.assertEquals(expected.getClass(), evaluated.getClass());
final Expr parsedNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
Expr roundTrip = Parser.parse(parsedNoFlatten.stringify(), ExprMacroTable.nil());
Assert.assertArrayEquals(
expression,
expected,
(Object[]) roundTrip.eval(Parser.withMap(ImmutableMap.of())).value()
);
Assert.assertEquals(parsed.stringify(), roundTrip.stringify());
} }
} }

View File

@ -37,28 +37,15 @@ This expression language supports the following operators (listed in decreasing
|<, <=, >, >=, ==, !=|Binary Comparison| |<, <=, >, >=, ==, !=|Binary Comparison|
|&&, &#124;&#124;|Binary Logical AND, OR| |&&, &#124;&#124;|Binary Logical AND, OR|
Long, double, and string data types are supported. If a number contains a dot, it is interpreted as a double, otherwise Long, double, and string data types are supported. If a number contains a dot, it is interpreted as a double, otherwise it is interpreted as a long. That means, always add a '.' to your number if you want it interpreted as a double value. String literals should be quoted by single quotation marks.
it is interpreted as a long. That means, always add a '.' to your number if you want it interpreted as a double value.
String literals should be quoted by single quotation marks.
Additionally, the expression language supports long, double, and string arrays. Array literals are created by wrapping Additionally, the expression language supports long, double, and string arrays. Array literals are created by wrapping square brackets around a list of scalar literals values delimited by a comma or space character. All values in an array literal must be the same type, however null values are accepted. Typed empty arrays may be defined by prefixing with their type in angle brackets: `<STRING>[]`, `<DOUBLE>[]`, or `<LONG>[]`.
square brackets around a list of scalar literals values delimited by a comma or space character. All values in an array
literal must be the same type.
Expressions can contain variables. Variable names may contain letters, digits, '\_' and '$'. Variable names must not Expressions can contain variables. Variable names may contain letters, digits, '\_' and '$'. Variable names must not begin with a digit. To escape other special characters, you can quote it with double quotation marks.
begin with a digit. To escape other special characters, you can quote it with double quotation marks.
For logical operators, a number is true if and only if it is positive (0 or negative value means false). For string For logical operators, a number is true if and only if it is positive (0 or negative value means false). For string type, it's the evaluation result of 'Boolean.valueOf(string)'.
type, it's the evaluation result of 'Boolean.valueOf(string)'.
[Multi-value string dimensions](../querying/multi-value-dimensions.html) are supported and may be treated as either [Multi-value string dimensions](../querying/multi-value-dimensions.html) are supported and may be treated as either scalar or array typed values. When treated as a scalar type, an expression will automatically be transformed to apply the scalar operation across all values of the multi-valued type, to mimic Druid's native behavior. Values that result in arrays will be coerced back into the native Druid string type for aggregation. Druid aggregations on multi-value string dimensions on the individual values, _not_ the 'array', behaving similar to the `UNNEST` operator available in many SQL dialects. However, by using the `array_to_string` function, aggregations may be done on a stringified version of the complete array, allowing the complete row to be preserved. Using `string_to_array` in an expression post-aggregator, allows transforming the stringified dimension back into the true native array type.
scalar or array typed values. When treated as a scalar type, an expression will automatically be transformed to apply
the scalar operation across all values of the multi-valued type, to mimic Druid's native behavior. Values that result in
arrays will be coerced back into the native Druid string type for aggregation. Druid aggregations on multi-value string
dimensions on the individual values, _not_ the 'array', behaving similar to the `UNNEST` operator available in many SQL
dialects. However, by using the `array_to_string` function, aggregations may be done on a stringified version of the
complete array, allowing the complete row to be preserved. Using `string_to_array` in an expression post-aggregator,
allows transforming the stringified dimension back into the true native array type.
The following built-in functions are available. The following built-in functions are available.
@ -196,10 +183,7 @@ See javadoc of java.lang.Math for detailed explanation for each function.
## IP address functions ## IP address functions
For the IPv4 address functions, the `address` argument can either be an IPv4 dotted-decimal string For the IPv4 address functions, the `address` argument can either be an IPv4 dotted-decimal string (e.g., "192.168.0.1") or an IP address represented as a long (e.g., 3232235521). The `subnet` argument should be a string formatted as an IPv4 address subnet in CIDR notation (e.g., "192.168.0.0/16").
(e.g., "192.168.0.1") or an IP address represented as a long (e.g., 3232235521). The `subnet`
argument should be a string formatted as an IPv4 address subnet in CIDR notation (e.g.,
"192.168.0.0/16").
| function | description | | function | description |
| --- | --- | | --- | --- |

View File

@ -71,7 +71,7 @@ public class BloomFilterExprMacro implements ExprMacroTable.ExprMacro
{ {
private BloomExpr(Expr arg) private BloomExpr(Expr arg)
{ {
super(arg); super(FN_NAME, arg);
} }
@Nonnull @Nonnull

View File

@ -21,6 +21,7 @@ package org.apache.druid.query.expression;
import org.apache.commons.net.util.SubnetUtils; import org.apache.commons.net.util.SubnetUtils;
import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval; import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.ExprMacroTable;
@ -52,13 +53,13 @@ import java.util.List;
*/ */
public class IPv4AddressMatchExprMacro implements ExprMacroTable.ExprMacro public class IPv4AddressMatchExprMacro implements ExprMacroTable.ExprMacro
{ {
public static final String NAME = "ipv4_match"; public static final String FN_NAME = "ipv4_match";
private static final int ARG_SUBNET = 1; private static final int ARG_SUBNET = 1;
@Override @Override
public String name() public String name()
{ {
return NAME; return FN_NAME;
} }
@Override @Override
@ -77,7 +78,7 @@ public class IPv4AddressMatchExprMacro implements ExprMacroTable.ExprMacro
private IPv4AddressMatchExpr(Expr arg, SubnetUtils.SubnetInfo subnetInfo) private IPv4AddressMatchExpr(Expr arg, SubnetUtils.SubnetInfo subnetInfo)
{ {
super(arg); super(FN_NAME, arg);
this.subnetInfo = subnetInfo; this.subnetInfo = subnetInfo;
} }
@ -116,6 +117,12 @@ public class IPv4AddressMatchExprMacro implements ExprMacroTable.ExprMacro
Expr newArg = arg.visit(shuttle); Expr newArg = arg.visit(shuttle);
return shuttle.visit(new IPv4AddressMatchExpr(newArg, subnetInfo)); return shuttle.visit(new IPv4AddressMatchExpr(newArg, subnetInfo));
} }
@Override
public String stringify()
{
return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), args.get(ARG_SUBNET).stringify());
}
} }
return new IPv4AddressMatchExpr(arg, subnetInfo); return new IPv4AddressMatchExpr(arg, subnetInfo);

View File

@ -47,12 +47,12 @@ import java.util.List;
*/ */
public class IPv4AddressParseExprMacro implements ExprMacroTable.ExprMacro public class IPv4AddressParseExprMacro implements ExprMacroTable.ExprMacro
{ {
public static final String NAME = "ipv4_parse"; public static final String FN_NAME = "ipv4_parse";
@Override @Override
public String name() public String name()
{ {
return NAME; return FN_NAME;
} }
@Override @Override
@ -68,7 +68,7 @@ public class IPv4AddressParseExprMacro implements ExprMacroTable.ExprMacro
{ {
private IPv4AddressParseExpr(Expr arg) private IPv4AddressParseExpr(Expr arg)
{ {
super(arg); super(FN_NAME, arg);
} }
@Nonnull @Nonnull

View File

@ -46,12 +46,12 @@ import java.util.List;
*/ */
public class IPv4AddressStringifyExprMacro implements ExprMacroTable.ExprMacro public class IPv4AddressStringifyExprMacro implements ExprMacroTable.ExprMacro
{ {
public static final String NAME = "ipv4_stringify"; public static final String FN_NAME = "ipv4_stringify";
@Override @Override
public String name() public String name()
{ {
return NAME; return FN_NAME;
} }
@Override @Override
@ -67,7 +67,7 @@ public class IPv4AddressStringifyExprMacro implements ExprMacroTable.ExprMacro
{ {
private IPv4AddressStringifyExpr(Expr arg) private IPv4AddressStringifyExpr(Expr arg)
{ {
super(arg); super(FN_NAME, arg);
} }
@Nonnull @Nonnull

View File

@ -21,6 +21,7 @@ package org.apache.druid.query.expression;
import org.apache.druid.common.config.NullHandling; import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval; import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.ExprMacroTable;
@ -32,10 +33,12 @@ import java.util.List;
public class LikeExprMacro implements ExprMacroTable.ExprMacro public class LikeExprMacro implements ExprMacroTable.ExprMacro
{ {
private static final String FN_NAME = "like";
@Override @Override
public String name() public String name()
{ {
return "like"; return FN_NAME;
} }
@Override @Override
@ -71,7 +74,7 @@ public class LikeExprMacro implements ExprMacroTable.ExprMacro
{ {
private LikeExtractExpr(Expr arg) private LikeExtractExpr(Expr arg)
{ {
super(arg); super(FN_NAME, arg);
} }
@Nonnull @Nonnull
@ -87,6 +90,21 @@ public class LikeExprMacro implements ExprMacroTable.ExprMacro
Expr newArg = arg.visit(shuttle); Expr newArg = arg.visit(shuttle);
return shuttle.visit(new LikeExtractExpr(newArg)); return shuttle.visit(new LikeExtractExpr(newArg));
} }
@Override
public String stringify()
{
if (escapeExpr != null) {
return StringUtils.format(
"%s(%s, %s, %s)",
FN_NAME,
arg.stringify(),
patternExpr.stringify(),
escapeExpr.stringify()
);
}
return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), patternExpr.stringify());
}
} }
return new LikeExtractExpr(arg); return new LikeExtractExpr(arg);
} }

View File

@ -22,6 +22,7 @@ package org.apache.druid.query.expression;
import com.google.inject.Inject; import com.google.inject.Inject;
import org.apache.druid.common.config.NullHandling; import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval; import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.ExprMacroTable;
@ -33,6 +34,7 @@ import java.util.List;
public class LookupExprMacro implements ExprMacroTable.ExprMacro public class LookupExprMacro implements ExprMacroTable.ExprMacro
{ {
private static final String FN_NAME = "lookup";
private final LookupExtractorFactoryContainerProvider lookupExtractorFactoryContainerProvider; private final LookupExtractorFactoryContainerProvider lookupExtractorFactoryContainerProvider;
@Inject @Inject
@ -44,7 +46,7 @@ public class LookupExprMacro implements ExprMacroTable.ExprMacro
@Override @Override
public String name() public String name()
{ {
return "lookup"; return FN_NAME;
} }
@Override @Override
@ -75,7 +77,7 @@ public class LookupExprMacro implements ExprMacroTable.ExprMacro
{ {
private LookupExpr(Expr arg) private LookupExpr(Expr arg)
{ {
super(arg); super(FN_NAME, arg);
} }
@Nonnull @Nonnull
@ -91,6 +93,12 @@ public class LookupExprMacro implements ExprMacroTable.ExprMacro
Expr newArg = arg.visit(shuttle); Expr newArg = arg.visit(shuttle);
return shuttle.visit(new LookupExpr(newArg)); return shuttle.visit(new LookupExpr(newArg));
} }
@Override
public String stringify()
{
return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), lookupExpr.stringify());
}
} }
return new LookupExpr(arg); return new LookupExpr(arg);

View File

@ -21,6 +21,7 @@ package org.apache.druid.query.expression;
import org.apache.druid.common.config.NullHandling; import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval; import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.ExprMacroTable;
@ -32,10 +33,12 @@ import java.util.regex.Pattern;
public class RegexpExtractExprMacro implements ExprMacroTable.ExprMacro public class RegexpExtractExprMacro implements ExprMacroTable.ExprMacro
{ {
private static final String FN_NAME = "regexp_extract";
@Override @Override
public String name() public String name()
{ {
return "regexp_extract"; return FN_NAME;
} }
@Override @Override
@ -62,7 +65,7 @@ public class RegexpExtractExprMacro implements ExprMacroTable.ExprMacro
{ {
private RegexpExtractExpr(Expr arg) private RegexpExtractExpr(Expr arg)
{ {
super(arg); super(FN_NAME, arg);
} }
@Nonnull @Nonnull
@ -81,6 +84,21 @@ public class RegexpExtractExprMacro implements ExprMacroTable.ExprMacro
Expr newArg = arg.visit(shuttle); Expr newArg = arg.visit(shuttle);
return shuttle.visit(new RegexpExtractExpr(newArg)); return shuttle.visit(new RegexpExtractExpr(newArg));
} }
@Override
public String stringify()
{
if (indexExpr != null) {
return StringUtils.format(
"%s(%s, %s, %s)",
FN_NAME,
arg.stringify(),
patternExpr.stringify(),
indexExpr.stringify()
);
}
return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), patternExpr.stringify());
}
} }
return new RegexpExtractExpr(arg); return new RegexpExtractExpr(arg);
} }

View File

@ -34,10 +34,12 @@ import java.util.stream.Collectors;
public class TimestampCeilExprMacro implements ExprMacroTable.ExprMacro public class TimestampCeilExprMacro implements ExprMacroTable.ExprMacro
{ {
private static final String FN_NAME = "timestamp_ceil";
@Override @Override
public String name() public String name()
{ {
return "timestamp_ceil"; return FN_NAME;
} }
@Override @Override
@ -60,7 +62,7 @@ public class TimestampCeilExprMacro implements ExprMacroTable.ExprMacro
TimestampCeilExpr(final List<Expr> args) TimestampCeilExpr(final List<Expr> args)
{ {
super(args); super(FN_NAME, args);
this.granularity = getGranularity(args, ExprUtils.nilBindings()); this.granularity = getGranularity(args, ExprUtils.nilBindings());
} }
@ -103,7 +105,7 @@ public class TimestampCeilExprMacro implements ExprMacroTable.ExprMacro
{ {
TimestampCeilDynamicExpr(final List<Expr> args) TimestampCeilDynamicExpr(final List<Expr> args)
{ {
super(args); super(FN_NAME, args);
} }
@Nonnull @Nonnull

View File

@ -34,6 +34,8 @@ import java.util.List;
public class TimestampExtractExprMacro implements ExprMacroTable.ExprMacro public class TimestampExtractExprMacro implements ExprMacroTable.ExprMacro
{ {
private static final String FN_NAME = "timestamp_extract";
public enum Unit public enum Unit
{ {
EPOCH, EPOCH,
@ -59,7 +61,7 @@ public class TimestampExtractExprMacro implements ExprMacroTable.ExprMacro
@Override @Override
public String name() public String name()
{ {
return "timestamp_extract"; return FN_NAME;
} }
@Override @Override
@ -93,7 +95,7 @@ public class TimestampExtractExprMacro implements ExprMacroTable.ExprMacro
{ {
private TimestampExtractExpr(Expr arg) private TimestampExtractExpr(Expr arg)
{ {
super(arg); super(FN_NAME, arg);
} }
@Nonnull @Nonnull
@ -159,6 +161,21 @@ public class TimestampExtractExprMacro implements ExprMacroTable.ExprMacro
Expr newArg = arg.visit(shuttle); Expr newArg = arg.visit(shuttle);
return shuttle.visit(new TimestampExtractExpr(newArg)); return shuttle.visit(new TimestampExtractExpr(newArg));
} }
@Override
public String stringify()
{
if (args.size() > 2) {
return StringUtils.format(
"%s(%s, %s, %s)",
FN_NAME,
arg.stringify(),
args.get(1).stringify(),
args.get(2).stringify()
);
}
return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), args.get(1).stringify());
}
} }
return new TimestampExtractExpr(arg); return new TimestampExtractExpr(arg);

View File

@ -32,10 +32,12 @@ import java.util.stream.Collectors;
public class TimestampFloorExprMacro implements ExprMacroTable.ExprMacro public class TimestampFloorExprMacro implements ExprMacroTable.ExprMacro
{ {
private static final String FN_NAME = "timestamp_floor";
@Override @Override
public String name() public String name()
{ {
return "timestamp_floor"; return FN_NAME;
} }
@Override @Override
@ -68,7 +70,7 @@ public class TimestampFloorExprMacro implements ExprMacroTable.ExprMacro
TimestampFloorExpr(final List<Expr> args) TimestampFloorExpr(final List<Expr> args)
{ {
super(args); super(FN_NAME, args);
this.granularity = computeGranularity(args, ExprUtils.nilBindings()); this.granularity = computeGranularity(args, ExprUtils.nilBindings());
} }
@ -113,7 +115,7 @@ public class TimestampFloorExprMacro implements ExprMacroTable.ExprMacro
{ {
TimestampFloorDynamicExpr(final List<Expr> args) TimestampFloorDynamicExpr(final List<Expr> args)
{ {
super(args); super(FN_NAME, args);
} }
@Nonnull @Nonnull

View File

@ -21,6 +21,7 @@ package org.apache.druid.query.expression;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval; import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.ExprMacroTable;
@ -34,10 +35,12 @@ import java.util.List;
public class TimestampFormatExprMacro implements ExprMacroTable.ExprMacro public class TimestampFormatExprMacro implements ExprMacroTable.ExprMacro
{ {
private static final String FN_NAME = "timestamp_format";
@Override @Override
public String name() public String name()
{ {
return "timestamp_format"; return FN_NAME;
} }
@Override @Override
@ -72,7 +75,7 @@ public class TimestampFormatExprMacro implements ExprMacroTable.ExprMacro
{ {
private TimestampFormatExpr(Expr arg) private TimestampFormatExpr(Expr arg)
{ {
super(arg); super(FN_NAME, arg);
} }
@Nonnull @Nonnull
@ -93,6 +96,24 @@ public class TimestampFormatExprMacro implements ExprMacroTable.ExprMacro
Expr newArg = arg.visit(shuttle); Expr newArg = arg.visit(shuttle);
return shuttle.visit(new TimestampFormatExpr(newArg)); return shuttle.visit(new TimestampFormatExpr(newArg));
} }
@Override
public String stringify()
{
if (args.size() > 2) {
return StringUtils.format(
"%s(%s, %s, %s)",
FN_NAME,
arg.stringify(),
args.get(1).stringify(),
args.get(2).stringify()
);
}
if (args.size() > 1) {
return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), args.get(1).stringify());
}
return super.stringify();
}
} }
return new TimestampFormatExpr(arg); return new TimestampFormatExpr(arg);

View File

@ -21,6 +21,7 @@ package org.apache.druid.query.expression;
import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval; import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.ExprMacroTable;
@ -36,10 +37,12 @@ import java.util.List;
public class TimestampParseExprMacro implements ExprMacroTable.ExprMacro public class TimestampParseExprMacro implements ExprMacroTable.ExprMacro
{ {
private static final String FN_NAME = "timestamp_parse";
@Override @Override
public String name() public String name()
{ {
return "timestamp_parse"; return FN_NAME;
} }
@Override @Override
@ -68,7 +71,7 @@ public class TimestampParseExprMacro implements ExprMacroTable.ExprMacro
{ {
private TimestampParseExpr(Expr arg) private TimestampParseExpr(Expr arg)
{ {
super(arg); super(FN_NAME, arg);
} }
@Nonnull @Nonnull
@ -96,6 +99,24 @@ public class TimestampParseExprMacro implements ExprMacroTable.ExprMacro
Expr newArg = arg.visit(shuttle); Expr newArg = arg.visit(shuttle);
return shuttle.visit(new TimestampParseExpr(newArg)); return shuttle.visit(new TimestampParseExpr(newArg));
} }
@Override
public String stringify()
{
if (args.size() > 2) {
return StringUtils.format(
"%s(%s, %s, %s)",
FN_NAME,
arg.stringify(),
args.get(1).stringify(),
args.get(2).stringify()
);
}
if (args.size() > 1) {
return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), args.get(1).stringify());
}
return super.stringify();
}
} }
return new TimestampParseExpr(arg); return new TimestampParseExpr(arg);

View File

@ -34,10 +34,12 @@ import java.util.stream.Collectors;
public class TimestampShiftExprMacro implements ExprMacroTable.ExprMacro public class TimestampShiftExprMacro implements ExprMacroTable.ExprMacro
{ {
private static final String FN_NAME = "timestamp_shift";
@Override @Override
public String name() public String name()
{ {
return "timestamp_shift"; return FN_NAME;
} }
@Override @Override
@ -79,7 +81,7 @@ public class TimestampShiftExprMacro implements ExprMacroTable.ExprMacro
TimestampShiftExpr(final List<Expr> args) TimestampShiftExpr(final List<Expr> args)
{ {
super(args); super(FN_NAME, args);
final PeriodGranularity granularity = getGranularity(args, ExprUtils.nilBindings()); final PeriodGranularity granularity = getGranularity(args, ExprUtils.nilBindings());
period = granularity.getPeriod(); period = granularity.getPeriod();
chronology = ISOChronology.getInstance(granularity.getTimeZone()); chronology = ISOChronology.getInstance(granularity.getTimeZone());
@ -105,7 +107,7 @@ public class TimestampShiftExprMacro implements ExprMacroTable.ExprMacro
{ {
TimestampShiftDynamicExpr(final List<Expr> args) TimestampShiftDynamicExpr(final List<Expr> args)
{ {
super(args); super(FN_NAME, args);
} }
@Nonnull @Nonnull

View File

@ -21,6 +21,7 @@ package org.apache.druid.query.expression;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval; import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.ExprMacroTable;
@ -35,19 +36,26 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
enum TrimMode enum TrimMode
{ {
BOTH(true, true), BOTH("trim", true, true),
LEFT(true, false), LEFT("ltrim", true, false),
RIGHT(false, true); RIGHT("rtrim", false, true);
private final String name;
private final boolean left; private final boolean left;
private final boolean right; private final boolean right;
TrimMode(final boolean left, final boolean right) TrimMode(final String name, final boolean left, final boolean right)
{ {
this.name = name;
this.left = left; this.left = left;
this.right = right; this.right = right;
} }
public String getFnName()
{
return name;
}
public boolean isLeft() public boolean isLeft()
{ {
return left; return left;
@ -60,18 +68,16 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
} }
private final TrimMode mode; private final TrimMode mode;
private final String name;
public TrimExprMacro(final String name, final TrimMode mode) public TrimExprMacro(final TrimMode mode)
{ {
this.name = name;
this.mode = mode; this.mode = mode;
} }
@Override @Override
public String name() public String name()
{ {
return name; return mode.getFnName();
} }
@Override @Override
@ -82,13 +88,13 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
} }
if (args.size() == 1) { if (args.size() == 1) {
return new TrimStaticCharsExpr(mode, args.get(0), DEFAULT_CHARS); return new TrimStaticCharsExpr(mode, args.get(0), DEFAULT_CHARS, null);
} else { } else {
final Expr charsArg = args.get(1); final Expr charsArg = args.get(1);
if (charsArg.isLiteral()) { if (charsArg.isLiteral()) {
final String charsString = charsArg.eval(ExprUtils.nilBindings()).asString(); final String charsString = charsArg.eval(ExprUtils.nilBindings()).asString();
final char[] chars = charsString == null ? EMPTY_CHARS : charsString.toCharArray(); final char[] chars = charsString == null ? EMPTY_CHARS : charsString.toCharArray();
return new TrimStaticCharsExpr(mode, args.get(0), chars); return new TrimStaticCharsExpr(mode, args.get(0), chars, charsArg);
} else { } else {
return new TrimDynamicCharsExpr(mode, args.get(0), args.get(1)); return new TrimDynamicCharsExpr(mode, args.get(0), args.get(1));
} }
@ -99,12 +105,14 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
{ {
private final TrimMode mode; private final TrimMode mode;
private final char[] chars; private final char[] chars;
private final Expr charsExpr;
public TrimStaticCharsExpr(final TrimMode mode, final Expr stringExpr, final char[] chars) public TrimStaticCharsExpr(final TrimMode mode, final Expr stringExpr, final char[] chars, final Expr charsExpr)
{ {
super(stringExpr); super(mode.getFnName(), stringExpr);
this.mode = mode; this.mode = mode;
this.chars = chars; this.chars = chars;
this.charsExpr = charsExpr;
} }
@Nonnull @Nonnull
@ -153,7 +161,16 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
public Expr visit(Shuttle shuttle) public Expr visit(Shuttle shuttle)
{ {
Expr newStringExpr = arg.visit(shuttle); Expr newStringExpr = arg.visit(shuttle);
return shuttle.visit(new TrimStaticCharsExpr(mode, newStringExpr, chars)); return shuttle.visit(new TrimStaticCharsExpr(mode, newStringExpr, chars, charsExpr));
}
@Override
public String stringify()
{
if (charsExpr != null) {
return StringUtils.format("%s(%s, %s)", mode.getFnName(), arg.stringify(), charsExpr.stringify());
}
return super.stringify();
} }
} }
@ -219,6 +236,12 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
} }
} }
@Override
public String stringify()
{
return StringUtils.format("%s(%s, %s)", mode.getFnName(), stringExpr.stringify(), charsExpr.stringify());
}
@Override @Override
public void visit(final Visitor visitor) public void visit(final Visitor visitor)
{ {
@ -270,7 +293,7 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
{ {
public BothTrimExprMacro() public BothTrimExprMacro()
{ {
super("trim", TrimMode.BOTH); super(TrimMode.BOTH);
} }
} }
@ -278,7 +301,7 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
{ {
public LeftTrimExprMacro() public LeftTrimExprMacro()
{ {
super("ltrim", TrimMode.LEFT); super(TrimMode.LEFT);
} }
} }
@ -286,7 +309,7 @@ public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
{ {
public RightTrimExprMacro() public RightTrimExprMacro()
{ {
super("rtrim", TrimMode.RIGHT); super(TrimMode.RIGHT);
} }
} }
} }

View File

@ -193,7 +193,7 @@ public class IPv4AddressMatchExprMacroTest extends MacroTestBase
{ {
NotLiteralExpr(Expr arg) NotLiteralExpr(Expr arg)
{ {
super(arg); super("not", arg);
} }
@Override @Override

View File

@ -232,5 +232,13 @@ public class ExprMacroTest
{ {
final Expr expr = Parser.parse(expression, LookupEnabledTestExprMacroTable.INSTANCE); final Expr expr = Parser.parse(expression, LookupEnabledTestExprMacroTable.INSTANCE);
Assert.assertEquals(expression, expectedResult, expr.eval(BINDINGS).value()); Assert.assertEquals(expression, expectedResult, expr.eval(BINDINGS).value());
final Expr exprNotFlattened = Parser.parse(expression, LookupEnabledTestExprMacroTable.INSTANCE, false);
final Expr roundTripNotFlattened =
Parser.parse(exprNotFlattened.stringify(), LookupEnabledTestExprMacroTable.INSTANCE);
Assert.assertEquals(exprNotFlattened.stringify(), expectedResult, roundTripNotFlattened.eval(BINDINGS).value());
final Expr roundTrip = Parser.parse(expr.stringify(), LookupEnabledTestExprMacroTable.INSTANCE);
Assert.assertEquals(exprNotFlattened.stringify(), expectedResult, roundTrip.eval(BINDINGS).value());
} }
} }

View File

@ -41,7 +41,7 @@ public class IPv4AddressMatchOperatorConversion extends DirectOperatorConversion
private static final SqlSingleOperandTypeChecker SUBNET_OPERAND = OperandTypes.family(SqlTypeFamily.STRING); private static final SqlSingleOperandTypeChecker SUBNET_OPERAND = OperandTypes.family(SqlTypeFamily.STRING);
private static final SqlFunction SQL_FUNCTION = OperatorConversions private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder(StringUtils.toUpperCase(IPv4AddressMatchExprMacro.NAME)) .operatorBuilder(StringUtils.toUpperCase(IPv4AddressMatchExprMacro.FN_NAME))
.operandTypeChecker(OperandTypes.sequence("(expr,string)", ADDRESS_OPERAND, SUBNET_OPERAND)) .operandTypeChecker(OperandTypes.sequence("(expr,string)", ADDRESS_OPERAND, SUBNET_OPERAND))
.returnTypeInference(ReturnTypes.BOOLEAN_NULLABLE) .returnTypeInference(ReturnTypes.BOOLEAN_NULLABLE)
.functionCategory(SqlFunctionCategory.USER_DEFINED_FUNCTION) .functionCategory(SqlFunctionCategory.USER_DEFINED_FUNCTION)
@ -49,7 +49,7 @@ public class IPv4AddressMatchOperatorConversion extends DirectOperatorConversion
public IPv4AddressMatchOperatorConversion() public IPv4AddressMatchOperatorConversion()
{ {
super(SQL_FUNCTION, IPv4AddressMatchExprMacro.NAME); super(SQL_FUNCTION, IPv4AddressMatchExprMacro.FN_NAME);
} }
@Override @Override

View File

@ -33,7 +33,7 @@ import org.apache.druid.sql.calcite.expression.OperatorConversions;
public class IPv4AddressParseOperatorConversion extends DirectOperatorConversion public class IPv4AddressParseOperatorConversion extends DirectOperatorConversion
{ {
private static final SqlFunction SQL_FUNCTION = OperatorConversions private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder(StringUtils.toUpperCase(IPv4AddressParseExprMacro.NAME)) .operatorBuilder(StringUtils.toUpperCase(IPv4AddressParseExprMacro.FN_NAME))
.operandTypeChecker( .operandTypeChecker(
OperandTypes.or( OperandTypes.or(
OperandTypes.family(SqlTypeFamily.STRING), OperandTypes.family(SqlTypeFamily.STRING),
@ -45,7 +45,7 @@ public class IPv4AddressParseOperatorConversion extends DirectOperatorConversion
public IPv4AddressParseOperatorConversion() public IPv4AddressParseOperatorConversion()
{ {
super(SQL_FUNCTION, IPv4AddressParseExprMacro.NAME); super(SQL_FUNCTION, IPv4AddressParseExprMacro.FN_NAME);
} }
@Override @Override

View File

@ -33,7 +33,7 @@ import org.apache.druid.sql.calcite.expression.OperatorConversions;
public class IPv4AddressStringifyOperatorConversion extends DirectOperatorConversion public class IPv4AddressStringifyOperatorConversion extends DirectOperatorConversion
{ {
private static final SqlFunction SQL_FUNCTION = OperatorConversions private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder(StringUtils.toUpperCase(IPv4AddressStringifyExprMacro.NAME)) .operatorBuilder(StringUtils.toUpperCase(IPv4AddressStringifyExprMacro.FN_NAME))
.operandTypeChecker( .operandTypeChecker(
OperandTypes.or( OperandTypes.or(
OperandTypes.family(SqlTypeFamily.INTEGER), OperandTypes.family(SqlTypeFamily.INTEGER),
@ -45,7 +45,7 @@ public class IPv4AddressStringifyOperatorConversion extends DirectOperatorConver
public IPv4AddressStringifyOperatorConversion() public IPv4AddressStringifyOperatorConversion()
{ {
super(SQL_FUNCTION, IPv4AddressStringifyExprMacro.NAME); super(SQL_FUNCTION, IPv4AddressStringifyExprMacro.FN_NAME);
} }