Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Expressions: Allow escapes in quoted identifiers. (#3735)

This commit is contained in:
Gian Merlino 2016-12-05 10:47:55 -08:00 committed by Nishant
parent b64e06704e
commit ff42058453
3 changed files with 100 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4
View File

@ -17,13 +17,13 @@ expr : ('-'|'!') expr # unaryOpExpr
fnArgs : expr (',' expr)* # functionArgs fnArgs : expr (',' expr)* # functionArgs
; ;
IDENTIFIER : [_$a-zA-Z][_$a-zA-Z0-9]* | '"' [_$a-zA-Z][_$a-zA-Z0-9]* '"'; IDENTIFIER : [_$a-zA-Z][_$a-zA-Z0-9]* | '"' (ESC | ~ [\"\\])* '"';
LONG : [0-9]+ ; LONG : [0-9]+ ;
DOUBLE : [0-9]+ '.' [0-9]* ; DOUBLE : [0-9]+ '.' [0-9]* ;
WS : [ \t\r\n]+ -> skip ; WS : [ \t\r\n]+ -> skip ;
STRING : '\'' (ESC | ~ [\'\\])* '\''; STRING : '\'' (ESC | ~ [\'\\])* '\'';
fragment ESC : '\\' ([\'\\/bfnrt] | UNICODE) ; fragment ESC : '\\' ([\'\"\\/bfnrt] | UNICODE) ;
fragment UNICODE : 'u' HEX HEX HEX HEX ; fragment UNICODE : 'u' HEX HEX HEX HEX ;
fragment HEX : [0-9a-fA-F] ; fragment HEX : [0-9a-fA-F] ;

2
common/src/main/java/io/druid/math/expr/ExprListenerImpl.java
View File

@ -301,7 +301,7 @@ public class ExprListenerImpl extends ExprBaseListener
{ {
String text = ctx.getText(); String text = ctx.getText();
if (text.charAt(0) == '"' && text.charAt(text.length() - 1) == '"') { if (text.charAt(0) == '"' && text.charAt(text.length() - 1) == '"') {
text = text.substring(1, text.length() - 1); text = StringEscapeUtils.unescapeJava(text.substring(1, text.length() - 1));
} }
nodes.put( nodes.put(
ctx, ctx,

162
common/src/test/java/io/druid/math/expr/ParserTest.java
View File

@ -19,9 +19,13 @@
package io.druid.math.expr; package io.druid.math.expr;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import java.util.List;
/** /**
*/ */
public class ParserTest public class ParserTest
@ -49,129 +53,157 @@ public class ParserTest
@Test @Test
public void testSimpleUnaryOps2() public void testSimpleUnaryOps2()
{ {
validate("-1", "-1", "-1"); validateFlatten("-1", "-1", "-1");
validate("--1", "--1", "1"); validateFlatten("--1", "--1", "1");
validate("-1+2", "(+ -1 2)", "1"); validateFlatten("-1+2", "(+ -1 2)", "1");
validate("-1*2", "(* -1 2)", "-2"); validateFlatten("-1*2", "(* -1 2)", "-2");
validate("-1^2", "(^ -1 2)", "1"); validateFlatten("-1^2", "(^ -1 2)", "1");
}
private void validateParser(String expression, String expected, String identifiers)
{
Assert.assertEquals(expected, Parser.parse(expression).toString());
Assert.assertEquals(identifiers, Parser.findRequiredBindings(expression).toString());
} }
@Test @Test
public void testSimpleLogicalOps1() public void testSimpleLogicalOps1()
{ {
validateParser("x>y", "(> x y)", "[x, y]"); validateParser("x>y", "(> x y)", ImmutableList.of("x", "y"));
validateParser("x<y", "(< x y)", "[x, y]"); validateParser("x<y", "(< x y)", ImmutableList.of("x", "y"));
validateParser("x<=y", "(<= x y)", "[x, y]"); validateParser("x<=y", "(<= x y)", ImmutableList.of("x", "y"));
validateParser("x>=y", "(>= x y)", "[x, y]"); validateParser("x>=y", "(>= x y)", ImmutableList.of("x", "y"));
validateParser("x==y", "(== x y)", "[x, y]"); validateParser("x==y", "(== x y)", ImmutableList.of("x", "y"));
validateParser("x!=y", "(!= x y)", "[x, y]"); validateParser("x!=y", "(!= x y)", ImmutableList.of("x", "y"));
validateParser("x && y", "(&& x y)", "[x, y]"); validateParser("x && y", "(&& x y)", ImmutableList.of("x", "y"));
validateParser("x || y", "(|| x y)", "[x, y]"); validateParser("x || y", "(|| x y)", ImmutableList.of("x", "y"));
} }
@Test @Test
public void testSimpleAdditivityOp1() public void testSimpleAdditivityOp1()
{ {
validateParser("x+y", "(+ x y)", "[x, y]"); validateParser("x+y", "(+ x y)", ImmutableList.of("x", "y"));
validateParser("x-y", "(- x y)", "[x, y]"); validateParser("x-y", "(- x y)", ImmutableList.of("x", "y"));
} }
@Test @Test
public void testSimpleAdditivityOp2() public void testSimpleAdditivityOp2()
{ {
validateParser("x+y+z", "(+ (+ x y) z)", "[x, y, z]"); validateParser("x+y+z", "(+ (+ x y) z)", ImmutableList.of("x", "y", "z"));
validateParser("x+y-z", "(- (+ x y) z)", "[x, y, z]"); validateParser("x+y-z", "(- (+ x y) z)", ImmutableList.of("x", "y", "z"));
validateParser("x-y+z", "(+ (- x y) z)", "[x, y, z]"); validateParser("x-y+z", "(+ (- x y) z)", ImmutableList.of("x", "y", "z"));
validateParser("x-y-z", "(- (- x y) z)", "[x, y, z]"); validateParser("x-y-z", "(- (- x y) z)", ImmutableList.of("x", "y", "z"));
validateParser("x-y-x", "(- (- x y) x)", "[x, y]"); validateParser("x-y-x", "(- (- x y) x)", ImmutableList.of("x", "y"));
} }
@Test @Test
public void testSimpleMultiplicativeOp1() public void testSimpleMultiplicativeOp1()
{ {
validateParser("x*y", "(* x y)", "[x, y]"); validateParser("x*y", "(* x y)", ImmutableList.of("x", "y"));
validateParser("x/y", "(/ x y)", "[x, y]"); validateParser("x/y", "(/ x y)", ImmutableList.of("x", "y"));
validateParser("x%y", "(% x y)", "[x, y]"); validateParser("x%y", "(% x y)", ImmutableList.of("x", "y"));
} }
@Test @Test
public void testSimpleMultiplicativeOp2() public void testSimpleMultiplicativeOp2()
{ {
validate("1*2*3", "(* (* 1 2) 3)", "6"); validateFlatten("1*2*3", "(* (* 1 2) 3)", "6");
validate("1*2/3", "(/ (* 1 2) 3)", "0"); validateFlatten("1*2/3", "(/ (* 1 2) 3)", "0");
validate("1/2*3", "(* (/ 1 2) 3)", "0"); validateFlatten("1/2*3", "(* (/ 1 2) 3)", "0");
validate("1/2/3", "(/ (/ 1 2) 3)", "0"); validateFlatten("1/2/3", "(/ (/ 1 2) 3)", "0");
validate("1.0*2*3", "(* (* 1.0 2) 3)", "6.0"); validateFlatten("1.0*2*3", "(* (* 1.0 2) 3)", "6.0");
validate("1.0*2/3", "(/ (* 1.0 2) 3)", "0.6666666666666666"); validateFlatten("1.0*2/3", "(/ (* 1.0 2) 3)", "0.6666666666666666");
validate("1.0/2*3", "(* (/ 1.0 2) 3)", "1.5"); validateFlatten("1.0/2*3", "(* (/ 1.0 2) 3)", "1.5");
validate("1.0/2/3", "(/ (/ 1.0 2) 3)", "0.16666666666666666"); validateFlatten("1.0/2/3", "(/ (/ 1.0 2) 3)", "0.16666666666666666");
// partial // partial
validate("1.0*2*x", "(* (* 1.0 2) x)", "(* 2.0 x)"); validateFlatten("1.0*2*x", "(* (* 1.0 2) x)", "(* 2.0 x)");
validate("1.0*2/x", "(/ (* 1.0 2) x)", "(/ 2.0 x)"); validateFlatten("1.0*2/x", "(/ (* 1.0 2) x)", "(/ 2.0 x)");
validate("1.0/2*x", "(* (/ 1.0 2) x)", "(* 0.5 x)"); validateFlatten("1.0/2*x", "(* (/ 1.0 2) x)", "(* 0.5 x)");
validate("1.0/2/x", "(/ (/ 1.0 2) x)", "(/ 0.5 x)"); validateFlatten("1.0/2/x", "(/ (/ 1.0 2) x)", "(/ 0.5 x)");
// not working yet // not working yet
validate("1.0*x*3", "(* (* 1.0 x) 3)", "(* (* 1.0 x) 3)"); validateFlatten("1.0*x*3", "(* (* 1.0 x) 3)", "(* (* 1.0 x) 3)");
} }
@Test @Test
public void testSimpleCarrot1() public void testSimpleCarrot1()
{ {
validate("1^2", "(^ 1 2)", "1"); validateFlatten("1^2", "(^ 1 2)", "1");
} }
@Test @Test
public void testSimpleCarrot2() public void testSimpleCarrot2()
{ {
validate("1^2^3", "(^ 1 (^ 2 3))", "1"); validateFlatten("1^2^3", "(^ 1 (^ 2 3))", "1");
} }
@Test @Test
public void testMixed() public void testMixed()
{ {
validate("1+2*3", "(+ 1 (* 2 3))", "7"); validateFlatten("1+2*3", "(+ 1 (* 2 3))", "7");
validate("1+(2*3)", "(+ 1 (* 2 3))", "7"); validateFlatten("1+(2*3)", "(+ 1 (* 2 3))", "7");
validate("(1+2)*3", "(* (+ 1 2) 3)", "9"); validateFlatten("(1+2)*3", "(* (+ 1 2) 3)", "9");
validate("1*2+3", "(+ (* 1 2) 3)", "5"); validateFlatten("1*2+3", "(+ (* 1 2) 3)", "5");
validate("(1*2)+3", "(+ (* 1 2) 3)", "5"); validateFlatten("(1*2)+3", "(+ (* 1 2) 3)", "5");
validate("1*(2+3)", "(* 1 (+ 2 3))", "5"); validateFlatten("1*(2+3)", "(* 1 (+ 2 3))", "5");
validate("1+2^3", "(+ 1 (^ 2 3))", "9"); validateFlatten("1+2^3", "(+ 1 (^ 2 3))", "9");
validate("1+(2^3)", "(+ 1 (^ 2 3))", "9"); validateFlatten("1+(2^3)", "(+ 1 (^ 2 3))", "9");
validate("(1+2)^3", "(^ (+ 1 2) 3)", "27"); validateFlatten("(1+2)^3", "(^ (+ 1 2) 3)", "27");
validate("1^2+3", "(+ (^ 1 2) 3)", "4"); validateFlatten("1^2+3", "(+ (^ 1 2) 3)", "4");
validate("(1^2)+3", "(+ (^ 1 2) 3)", "4"); validateFlatten("(1^2)+3", "(+ (^ 1 2) 3)", "4");
validate("1^(2+3)", "(^ 1 (+ 2 3))", "1"); validateFlatten("1^(2+3)", "(^ 1 (+ 2 3))", "1");
validate("1^2*3+4", "(+ (* (^ 1 2) 3) 4)", "7"); validateFlatten("1^2*3+4", "(+ (* (^ 1 2) 3) 4)", "7");
validate("-1^2*-3+-4", "(+ (* (^ -1 2) -3) -4)", "-7"); validateFlatten("-1^2*-3+-4", "(+ (* (^ -1 2) -3) -4)", "-7");
validate("max(3, 4)", "(max [3, 4])", "4"); validateFlatten("max(3, 4)", "(max [3, 4])", "4");
validate("min(1, max(3, 4))", "(min [1, (max [3, 4])])", "1"); validateFlatten("min(1, max(3, 4))", "(min [1, (max [3, 4])])", "1");
} }
private void validate(String expression, String withoutFlatten, String withFlatten) @Test
public void testIdentifiers()
{ {
Assert.assertEquals(withoutFlatten, Parser.parse(expression, false).toString()); validateParser("foo", "foo", ImmutableList.of("foo"));
Assert.assertEquals(withFlatten, Parser.parse(expression, true).toString()); validateParser("\"foo\"", "foo", ImmutableList.of("foo"));
validateParser("\"foo bar\"", "foo bar", ImmutableList.of("foo bar"));
validateParser("\"foo\\\"bar\"", "foo\"bar", ImmutableList.of("foo\"bar"));
}
@Test
public void testLiterals()
{
validateConstantExpression("\'foo\'", "foo");
validateConstantExpression("\'foo bar\'", "foo bar");
validateConstantExpression("\'föo bar\'", "föo bar");
validateConstantExpression("\'f\\u0040o bar\'", "f@o bar");
validateConstantExpression("\'f\\u000Ao \\'b\\\\\\\"ar\'", "f\no 'b\\\"ar");
} }
@Test @Test
public void testFunctions() public void testFunctions()
{ {
validateParser("sqrt(x)", "(sqrt [x])", "[x]"); validateParser("sqrt(x)", "(sqrt [x])", ImmutableList.of("x"));
validateParser("if(cond,then,else)", "(if [cond, then, else])", "[cond, then, else]"); validateParser("if(cond,then,else)", "(if [cond, then, else])", ImmutableList.of("cond", "then", "else"));
}
private void validateFlatten(String expression, String withoutFlatten, String withFlatten)
{
Assert.assertEquals(expression, withoutFlatten, Parser.parse(expression, false).toString());
Assert.assertEquals(expression, withFlatten, Parser.parse(expression, true).toString());
}
private void validateParser(String expression, String expected, List<String> identifiers)
{
Assert.assertEquals(expression, expected, Parser.parse(expression).toString());
Assert.assertEquals(expression, identifiers, Parser.findRequiredBindings(expression));
}
private void validateConstantExpression(String expression, Object expected)
{
Assert.assertEquals(
expression,
expected,
Parser.parse(expression).eval(Parser.withMap(ImmutableMap.<String, Object>of())).value()
);
} }
} }