Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2090 from himanshug/math_exp 

math expression support
This commit is contained in:
Fangjin Yang 2016-04-11 21:36:17 -07:00
parent 49ef4d96cb aa6a230c90
commit bd6bd34cd8
13 changed files with 1411 additions and 590 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
common/pom.xml
View File

@ -161,6 +161,10 @@
<artifactId>jets3t</artifactId> <artifactId>jets3t</artifactId>
<version>0.9.4</version> <version>0.9.4</version>
</dependency> </dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
</dependency>
<!-- Tests --> <!-- Tests -->
<dependency> <dependency>
@ -199,6 +203,17 @@
</execution> </execution>
</executions> </executions>
</plugin> </plugin>
<plugin>
<groupId>org.antlr</groupId>
<artifactId>antlr4-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>antlr4</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins> </plugins>
</build> </build>

38
common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4 Normal file
View File

@ -0,0 +1,38 @@
grammar Expr;
expr : ('-'|'!') expr # unaryOpExpr
|<assoc=right> expr '^' expr # powOpExpr
| expr ('*'|'/'|'%') expr # mulDivModuloExpr
| expr ('+'|'-') expr # addSubExpr
| expr ('<'|'<='|'>'|'>='|'=='|'!=') expr # logicalOpExpr
| expr ('&&'|'||') expr # logicalAndOrExpr
| '(' expr ')' # nestedExpr
| IDENTIFIER '(' fnArgs? ')' # functionExpr
| IDENTIFIER # identifierExpr
| DOUBLE # doubleExpr
| LONG # longExpr
;
fnArgs : expr (',' expr)* # functionArgs
;
IDENTIFIER : [_$a-zA-Z][_$a-zA-Z0-9]* ;
LONG : [0-9]+ ;
DOUBLE : [0-9]+ '.' [0-9]* ;
WS : [ \t\r\n]+ -> skip ;
MINUS : '-' ;
NOT : '!' ;
POW : '^' ;
MUL : '*' ;
DIV : '/' ;
MODULO : '%' ;
PLUS : '+' ;
LT : '<' ;
LEQ : '<=' ;
GT : '>' ;
GEQ : '>=' ;
EQ : '==' ;
NEQ : '!=' ;
AND : '&&' ;
OR : '||' ;

520
common/src/main/java/io/druid/math/expr/Expr.java Normal file
View File

@ -0,0 +1,520 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.math.expr;
import com.google.common.math.LongMath;
import java.util.List;
import java.util.Map;
/**
*/
public interface Expr
{
Number eval(Map<String, Number> bindings);
}
class LongExpr implements Expr
{
private final long value;
public LongExpr(long value)
{
this.value = value;
}
@Override
public String toString()
{
return String.valueOf(value);
}
@Override
public Number eval(Map<String, Number> bindings)
{
return value;
}
}
class DoubleExpr implements Expr
{
private final double value;
public DoubleExpr(double value)
{
this.value = value;
}
@Override
public String toString()
{
return String.valueOf(value);
}
@Override
public Number eval(Map<String, Number> bindings)
{
return value;
}
}
class IdentifierExpr implements Expr
{
private final String value;
public IdentifierExpr(String value)
{
this.value = value;
}
@Override
public String toString()
{
return value;
}
@Override
public Number eval(Map<String, Number> bindings)
{
Number val = bindings.get(value);
if (val == null) {
throw new RuntimeException("No binding found for " + value);
} else {
return val instanceof Long ? val : val.doubleValue();
}
}
}
class FunctionExpr implements Expr
{
private final String name;
private final List<Expr> args;
public FunctionExpr(String name, List<Expr> args)
{
this.name = name;
this.args = args;
}
@Override
public String toString()
{
return "(" + name + " " + args + ")";
}
@Override
public Number eval(Map<String, Number> bindings)
{
return Parser.func.get(name).apply(args, bindings);
}
}
class UnaryMinusExpr implements Expr
{
private final Expr expr;
UnaryMinusExpr(Expr expr)
{
this.expr = expr;
}
@Override
public Number eval(Map<String, Number> bindings)
{
Number valObj = expr.eval(bindings);
if (valObj instanceof Long) {
return -1 * valObj.longValue();
} else {
return -1 * valObj.doubleValue();
}
}
@Override
public String toString()
{
return "-" + expr.toString();
}
}
class UnaryNotExpr implements Expr
{
private final Expr expr;
UnaryNotExpr(Expr expr)
{
this.expr = expr;
}
@Override
public Number eval(Map<String, Number> bindings)
{
Number valObj = expr.eval(bindings);
return valObj.doubleValue() > 0 ? 0.0d : 1.0d;
}
@Override
public String toString()
{
return "!" + expr.toString();
}
}
abstract class BinaryOpExprBase implements Expr
{
protected final String op;
protected final Expr left;
protected final Expr right;
public BinaryOpExprBase(String op, Expr left, Expr right)
{
this.op = op;
this.left = left;
this.right = right;
}
protected boolean isLong(Number left, Number right)
{
return left instanceof Long && right instanceof Long;
}
@Override
public String toString()
{
return "(" + op + " " + left + " " + right + ")";
}
}
class BinMinusExpr extends BinaryOpExprBase
{
BinMinusExpr(String op, Expr left, Expr right)
{
super(op, left, right);
}
@Override
public Number eval(Map<String, Number> bindings)
{
Number leftVal = left.eval(bindings);
Number rightVal = right.eval(bindings);
if (isLong(leftVal, rightVal)) {
return leftVal.longValue() - rightVal.longValue();
} else {
return leftVal.doubleValue() - rightVal.doubleValue();
}
}
}
class BinPowExpr extends BinaryOpExprBase
{
BinPowExpr(String op, Expr left, Expr right)
{
super(op, left, right);
}
@Override
public Number eval(Map<String, Number> bindings)
{
Number leftVal = left.eval(bindings);
Number rightVal = right.eval(bindings);
if (isLong(leftVal, rightVal)) {
return LongMath.pow(leftVal.longValue(), rightVal.intValue());
} else {
return Math.pow(leftVal.doubleValue(), rightVal.doubleValue());
}
}
}
class BinMulExpr extends BinaryOpExprBase
{
BinMulExpr(String op, Expr left, Expr right)
{
super(op, left, right);
}
@Override
public Number eval(Map<String, Number> bindings)
{
Number leftVal = left.eval(bindings);
Number rightVal = right.eval(bindings);
if (isLong(leftVal, rightVal)) {
return leftVal.longValue() * rightVal.longValue();
} else {
return leftVal.doubleValue() * rightVal.doubleValue();
}
}
}
class BinDivExpr extends BinaryOpExprBase
{
BinDivExpr(String op, Expr left, Expr right)
{
super(op, left, right);
}
@Override
public Number eval(Map<String, Number> bindings)
{
Number leftVal = left.eval(bindings);
Number rightVal = right.eval(bindings);
if (isLong(leftVal, rightVal)) {
return leftVal.longValue() / rightVal.longValue();
} else {
return leftVal.doubleValue() / rightVal.doubleValue();
}
}
}
class BinModuloExpr extends BinaryOpExprBase
{
BinModuloExpr(String op, Expr left, Expr right)
{
super(op, left, right);
}
@Override
public Number eval(Map<String, Number> bindings)
{
Number leftVal = left.eval(bindings);
Number rightVal = right.eval(bindings);
if (isLong(leftVal, rightVal)) {
return leftVal.longValue() % rightVal.longValue();
} else {
return leftVal.doubleValue() % rightVal.doubleValue();
}
}
}
class BinPlusExpr extends BinaryOpExprBase
{
BinPlusExpr(String op, Expr left, Expr right)
{
super(op, left, right);
}
@Override
public Number eval(Map<String, Number> bindings)
{
Number leftVal = left.eval(bindings);
Number rightVal = right.eval(bindings);
if (isLong(leftVal, rightVal)) {
return leftVal.longValue() + rightVal.longValue();
} else {
return leftVal.doubleValue() + rightVal.doubleValue();
}
}
}
class BinLtExpr extends BinaryOpExprBase
{
BinLtExpr(String op, Expr left, Expr right)
{
super(op, left, right);
}
@Override
public Number eval(Map<String, Number> bindings)
{
Number leftVal = left.eval(bindings);
Number rightVal = right.eval(bindings);
if (isLong(leftVal, rightVal)) {
return leftVal.longValue() < rightVal.longValue() ? 1 : 0;
} else {
return leftVal.doubleValue() < rightVal.doubleValue() ? 1.0d : 0.0d;
}
}
}
class BinLeqExpr extends BinaryOpExprBase
{
BinLeqExpr(String op, Expr left, Expr right)
{
super(op, left, right);
}
@Override
public Number eval(Map<String, Number> bindings)
{
Number leftVal = left.eval(bindings);
Number rightVal = right.eval(bindings);
if (isLong(leftVal, rightVal)) {
return leftVal.longValue() <= rightVal.longValue() ? 1 : 0;
} else {
return leftVal.doubleValue() <= rightVal.doubleValue() ? 1.0d : 0.0d;
}
}
}
class BinGtExpr extends BinaryOpExprBase
{
BinGtExpr(String op, Expr left, Expr right)
{
super(op, left, right);
}
@Override
public Number eval(Map<String, Number> bindings)
{
Number leftVal = left.eval(bindings);
Number rightVal = right.eval(bindings);
if (isLong(leftVal, rightVal)) {
return leftVal.longValue() > rightVal.longValue() ? 1 : 0;
} else {
return leftVal.doubleValue() > rightVal.doubleValue() ? 1.0d : 0.0d;
}
}
}
class BinGeqExpr extends BinaryOpExprBase
{
BinGeqExpr(String op, Expr left, Expr right)
{
super(op, left, right);
}
@Override
public Number eval(Map<String, Number> bindings)
{
Number leftVal = left.eval(bindings);
Number rightVal = right.eval(bindings);
if (isLong(leftVal, rightVal)) {
return leftVal.longValue() >= rightVal.longValue() ? 1 : 0;
} else {
return leftVal.doubleValue() >= rightVal.doubleValue() ? 1.0d : 0.0d;
}
}
}
class BinEqExpr extends BinaryOpExprBase
{
BinEqExpr(String op, Expr left, Expr right)
{
super(op, left, right);
}
@Override
public Number eval(Map<String, Number> bindings)
{
Number leftVal = left.eval(bindings);
Number rightVal = right.eval(bindings);
if (isLong(leftVal, rightVal)) {
return leftVal.longValue() == rightVal.longValue() ? 1 : 0;
} else {
return leftVal.doubleValue() == rightVal.doubleValue() ? 1.0d : 0.0d;
}
}
}
class BinNeqExpr extends BinaryOpExprBase
{
BinNeqExpr(String op, Expr left, Expr right)
{
super(op, left, right);
}
@Override
public Number eval(Map<String, Number> bindings)
{
Number leftVal = left.eval(bindings);
Number rightVal = right.eval(bindings);
if (isLong(leftVal, rightVal)) {
return leftVal.longValue() != rightVal.longValue() ? 1 : 0;
} else {
return leftVal.doubleValue() != rightVal.doubleValue() ? 1.0d : 0.0d;
}
}
}
class BinAndExpr extends BinaryOpExprBase
{
BinAndExpr(String op, Expr left, Expr right)
{
super(op, left, right);
}
@Override
public Number eval(Map<String, Number> bindings)
{
Number leftVal = left.eval(bindings);
Number rightVal = right.eval(bindings);
if (isLong(leftVal, rightVal)) {
long lval = leftVal.longValue();
if (lval > 0) {
long rval = rightVal.longValue();
return rval > 0 ? 1 : 0;
} else {
return 0;
}
} else {
double lval = leftVal.doubleValue();
if (lval > 0) {
double rval = rightVal.doubleValue();
return rval > 0 ? 1.0d : 0.0d;
} else {
return 0.0d;
}
}
}
}
class BinOrExpr extends BinaryOpExprBase
{
BinOrExpr(String op, Expr left, Expr right)
{
super(op, left, right);
}
@Override
public Number eval(Map<String, Number> bindings)
{
Number leftVal = left.eval(bindings);
Number rightVal = right.eval(bindings);
if (isLong(leftVal, rightVal)) {
long lval = leftVal.longValue();
if (lval > 0) {
return 1;
} else {
long rval = rightVal.longValue();
return rval > 0 ? 1 : 0;
}
} else {
double lval = leftVal.doubleValue();
if (lval > 0) {
return 1.0d;
} else {
double rval = rightVal.doubleValue();
return rval > 0 ? 1.0d : 0.0d;
}
}
}
}

312
common/src/main/java/io/druid/math/expr/ExprListenerImpl.java Normal file
View File

@ -0,0 +1,312 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.math.expr;
import io.druid.math.expr.antlr.ExprBaseListener;
import io.druid.math.expr.antlr.ExprParser;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.TerminalNode;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
*/
public class ExprListenerImpl extends ExprBaseListener
{
private final Map<ParseTree, Object> nodes;
private final ParseTree rootNodeKey;
ExprListenerImpl(ParseTree rootNodeKey)
{
this.rootNodeKey = rootNodeKey;
this.nodes = new HashMap<>();
}
Expr getAST()
{
return (Expr) nodes.get(rootNodeKey);
}
@Override
public void exitUnaryOpExpr(ExprParser.UnaryOpExprContext ctx)
{
int opCode = ((TerminalNode) ctx.getChild(0)).getSymbol().getType();
switch (opCode) {
case ExprParser.MINUS:
nodes.put(ctx, new UnaryMinusExpr((Expr) nodes.get(ctx.getChild(1))));
break;
case ExprParser.NOT:
nodes.put(ctx, new UnaryNotExpr((Expr) nodes.get(ctx.getChild(1))));
break;
default:
throw new RuntimeException("Unrecognized unary operator " + ctx.getChild(0).getText());
}
}
@Override
public void exitDoubleExpr(ExprParser.DoubleExprContext ctx)
{
nodes.put(
ctx,
new DoubleExpr(Double.parseDouble(ctx.getText()))
);
}
@Override
public void exitAddSubExpr(ExprParser.AddSubExprContext ctx)
{
int opCode = ((TerminalNode) ctx.getChild(1)).getSymbol().getType();
switch (opCode) {
case ExprParser.PLUS:
nodes.put(
ctx,
new BinPlusExpr(
ctx.getChild(1).getText(),
(Expr) nodes.get(ctx.getChild(0)),
(Expr) nodes.get(ctx.getChild(2))
)
);
break;
case ExprParser.MINUS:
nodes.put(
ctx,
new BinMinusExpr(
ctx.getChild(1).getText(),
(Expr) nodes.get(ctx.getChild(0)),
(Expr) nodes.get(ctx.getChild(2))
)
);
break;
default:
throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText());
}
}
@Override
public void exitLongExpr(ExprParser.LongExprContext ctx)
{
nodes.put(
ctx,
new LongExpr(Long.parseLong(ctx.getText()))
);
}
@Override
public void exitLogicalAndOrExpr(ExprParser.LogicalAndOrExprContext ctx)
{
int opCode = ((TerminalNode) ctx.getChild(1)).getSymbol().getType();
switch (opCode) {
case ExprParser.AND:
nodes.put(
ctx,
new BinAndExpr(
ctx.getChild(1).getText(),
(Expr) nodes.get(ctx.getChild(0)),
(Expr) nodes.get(ctx.getChild(2))
)
);
break;
case ExprParser.OR:
nodes.put(
ctx,
new BinOrExpr(
ctx.getChild(1).getText(),
(Expr) nodes.get(ctx.getChild(0)),
(Expr) nodes.get(ctx.getChild(2))
)
);
break;
default:
throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText());
}
}
@Override
public void exitNestedExpr(ExprParser.NestedExprContext ctx)
{
nodes.put(ctx, nodes.get(ctx.getChild(1)));
}
@Override
public void exitLogicalOpExpr(ExprParser.LogicalOpExprContext ctx)
{
int opCode = ((TerminalNode) ctx.getChild(1)).getSymbol().getType();
switch (opCode) {
case ExprParser.LT:
nodes.put(
ctx,
new BinLtExpr(
ctx.getChild(1).getText(),
(Expr) nodes.get(ctx.getChild(0)),
(Expr) nodes.get(ctx.getChild(2))
)
);
break;
case ExprParser.LEQ:
nodes.put(
ctx,
new BinLeqExpr(
ctx.getChild(1).getText(),
(Expr) nodes.get(ctx.getChild(0)),
(Expr) nodes.get(ctx.getChild(2))
)
);
break;
case ExprParser.GT:
nodes.put(
ctx,
new BinGtExpr(
ctx.getChild(1).getText(),
(Expr) nodes.get(ctx.getChild(0)),
(Expr) nodes.get(ctx.getChild(2))
)
);
break;
case ExprParser.GEQ:
nodes.put(
ctx,
new BinGeqExpr(
ctx.getChild(1).getText(),
(Expr) nodes.get(ctx.getChild(0)),
(Expr) nodes.get(ctx.getChild(2))
)
);
break;
case ExprParser.EQ:
nodes.put(
ctx,
new BinEqExpr(
ctx.getChild(1).getText(),
(Expr) nodes.get(ctx.getChild(0)),
(Expr) nodes.get(ctx.getChild(2))
)
);
break;
case ExprParser.NEQ:
nodes.put(
ctx,
new BinNeqExpr(
ctx.getChild(1).getText(),
(Expr) nodes.get(ctx.getChild(0)),
(Expr) nodes.get(ctx.getChild(2))
)
);
break;
default:
throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText());
}
}
@Override
public void exitMulDivModuloExpr(ExprParser.MulDivModuloExprContext ctx)
{
int opCode = ((TerminalNode) ctx.getChild(1)).getSymbol().getType();
switch (opCode) {
case ExprParser.MUL:
nodes.put(
ctx,
new BinMulExpr(
ctx.getChild(1).getText(),
(Expr) nodes.get(ctx.getChild(0)),
(Expr) nodes.get(ctx.getChild(2))
)
);
break;
case ExprParser.DIV:
nodes.put(
ctx,
new BinDivExpr(
ctx.getChild(1).getText(),
(Expr) nodes.get(ctx.getChild(0)),
(Expr) nodes.get(ctx.getChild(2))
)
);
break;
case ExprParser.MODULO:
nodes.put(
ctx,
new BinModuloExpr(
ctx.getChild(1).getText(),
(Expr) nodes.get(ctx.getChild(0)),
(Expr) nodes.get(ctx.getChild(2))
)
);
break;
default:
throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText());
}
}
@Override
public void exitPowOpExpr(ExprParser.PowOpExprContext ctx)
{
nodes.put(
ctx,
new BinPowExpr(
ctx.getChild(1).getText(),
(Expr) nodes.get(ctx.getChild(0)),
(Expr) nodes.get(ctx.getChild(2))
)
);
}
@Override
public void exitFunctionExpr(ExprParser.FunctionExprContext ctx)
{
String fnName = ctx.getChild(0).getText();
if (!Parser.func.containsKey(fnName)) {
throw new RuntimeException("function " + fnName + " is not defined.");
}
List<Expr> args = ctx.getChildCount() > 3 ? (List<Expr>) nodes.get(ctx.getChild(2)) : Collections.<Expr>emptyList();
nodes.put(
ctx,
new FunctionExpr(fnName, args)
);
}
@Override
public void exitIdentifierExpr(ExprParser.IdentifierExprContext ctx)
{
nodes.put(
ctx,
new IdentifierExpr(ctx.getText())
);
}
@Override
public void exitFunctionArgs(ExprParser.FunctionArgsContext ctx)
{
List<Expr> args = new ArrayList<>();
args.add((Expr) nodes.get(ctx.getChild(0)));
if (ctx.getChildCount() > 1) {
for (int i = 1; i <= ctx.getChildCount() / 2; i++) {
args.add((Expr) nodes.get(ctx.getChild(2 * i)));
}
}
nodes.put(ctx, args);
}
}

64
common/src/main/java/io/druid/math/expr/Function.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.math.expr;
import java.util.List;
import java.util.Map;
/**
*/
interface Function
{
Number apply(List<Expr> args, Map<String, Number> bindings);
}
class SqrtFunc implements Function
{
@Override
public Number apply(List<Expr> args, Map<String, Number> bindings)
{
if (args.size() != 1) {
throw new RuntimeException("function 'sqrt' needs 1 argument");
}
Number x = args.get(0).eval(bindings);
return Math.sqrt(x.doubleValue());
}
}
class ConditionFunc implements Function
{
@Override
public Number apply(List<Expr> args, Map<String, Number> bindings)
{
if (args.size() != 3) {
throw new RuntimeException("function 'if' needs 3 argument");
}
Number x = args.get(0).eval(bindings);
if (x instanceof Long) {
return x.longValue() > 0 ? args.get(1).eval(bindings) : args.get(2).eval(bindings);
} else {
return x.doubleValue() > 0 ? args.get(1).eval(bindings) : args.get(2).eval(bindings);
}
}
}

56
common/src/main/java/io/druid/math/expr/Parser.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.math.expr;
import com.google.common.collect.ImmutableMap;
import io.druid.math.expr.antlr.ExprLexer;
import io.druid.math.expr.antlr.ExprParser;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.ParseTreeWalker;
import java.util.Map;
public class Parser
{
static final Map<String, Function> func;
static {
func = ImmutableMap.<String, Function>builder()
.put("sqrt", new SqrtFunc())
.put("if", new ConditionFunc())
.build();
}
public static Expr parse(String in)
{
ExprLexer lexer = new ExprLexer(new ANTLRInputStream(in));
CommonTokenStream tokens = new CommonTokenStream(lexer);
ExprParser parser = new ExprParser(tokens);
parser.setBuildParseTree(true);
ParseTree parseTree = parser.expr();
ParseTreeWalker walker = new ParseTreeWalker();
ExprListenerImpl listener = new ExprListenerImpl(parseTree);
walker.walk(listener, parseTree);
return listener.getAST();
}
}

109
common/src/test/java/io/druid/math/expr/EvalTest.java Normal file
View File

@ -0,0 +1,109 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.math.expr;
import org.junit.Assert;
import org.junit.Test;
import java.util.HashMap;
import java.util.Map;
/**
*/
public class EvalTest
{
@Test
public void testDoubleEval()
{
Map<String, Number> bindings = new HashMap<>();
bindings.put("x", 2.0d);
Assert.assertEquals(2.0, Parser.parse("x").eval(bindings).doubleValue(), 0.0001);
Assert.assertFalse(Parser.parse("1.0 && 0.0").eval(bindings).doubleValue() > 0.0);
Assert.assertTrue(Parser.parse("1.0 && 2.0").eval(bindings).doubleValue() > 0.0);
Assert.assertTrue(Parser.parse("1.0 || 0.0").eval(bindings).doubleValue() > 0.0);
Assert.assertFalse(Parser.parse("0.0 || 0.0").eval(bindings).doubleValue() > 0.0);
Assert.assertTrue(Parser.parse("2.0 > 1.0").eval(bindings).doubleValue() > 0.0);
Assert.assertTrue(Parser.parse("2.0 >= 2.0").eval(bindings).doubleValue() > 0.0);
Assert.assertTrue(Parser.parse("1.0 < 2.0").eval(bindings).doubleValue() > 0.0);
Assert.assertTrue(Parser.parse("2.0 <= 2.0").eval(bindings).doubleValue() > 0.0);
Assert.assertTrue(Parser.parse("2.0 == 2.0").eval(bindings).doubleValue() > 0.0);
Assert.assertTrue(Parser.parse("2.0 != 1.0").eval(bindings).doubleValue() > 0.0);
Assert.assertEquals(3.5, Parser.parse("2.0 + 1.5").eval(bindings).doubleValue(), 0.0001);
Assert.assertEquals(0.5, Parser.parse("2.0 - 1.5").eval(bindings).doubleValue(), 0.0001);
Assert.assertEquals(3.0, Parser.parse("2.0 * 1.5").eval(bindings).doubleValue(), 0.0001);
Assert.assertEquals(4.0, Parser.parse("2.0 / 0.5").eval(bindings).doubleValue(), 0.0001);
Assert.assertEquals(0.2, Parser.parse("2.0 % 0.3").eval(bindings).doubleValue(), 0.0001);
Assert.assertEquals(8.0, Parser.parse("2.0 ^ 3.0").eval(bindings).doubleValue(), 0.0001);
Assert.assertEquals(-1.5, Parser.parse("-1.5").eval(bindings).doubleValue(), 0.0001);
Assert.assertTrue(Parser.parse("!-1.0").eval(bindings).doubleValue() > 0.0);
Assert.assertTrue(Parser.parse("!0.0").eval(bindings).doubleValue() > 0.0);
Assert.assertFalse(Parser.parse("!2.0").eval(bindings).doubleValue() > 0.0);
Assert.assertEquals(2.0, Parser.parse("sqrt(4.0)").eval(bindings).doubleValue(), 0.0001);
Assert.assertEquals(2.0, Parser.parse("if(1.0, 2.0, 3.0)").eval(bindings).doubleValue(), 0.0001);
Assert.assertEquals(3.0, Parser.parse("if(0.0, 2.0, 3.0)").eval(bindings).doubleValue(), 0.0001);
}
@Test
public void testLongEval()
{
Map<String, Number> bindings = new HashMap<>();
bindings.put("x", 9223372036854775807L);
Assert.assertEquals(9223372036854775807L, Parser.parse("x").eval(bindings).longValue());
Assert.assertFalse(Parser.parse("9223372036854775807 && 0").eval(bindings).longValue() > 0);
Assert.assertTrue(Parser.parse("9223372036854775807 && 9223372036854775806").eval(bindings).longValue() > 0);
Assert.assertTrue(Parser.parse("9223372036854775807 || 0").eval(bindings).longValue() > 0);
Assert.assertFalse(Parser.parse("-9223372036854775807 || -9223372036854775807").eval(bindings).longValue() > 0);
Assert.assertTrue(Parser.parse("-9223372036854775807 || 9223372036854775807").eval(bindings).longValue() > 0);
Assert.assertFalse(Parser.parse("0 || 0").eval(bindings).longValue() > 0);
Assert.assertTrue(Parser.parse("9223372036854775807 > 9223372036854775806").eval(bindings).longValue() > 0);
Assert.assertTrue(Parser.parse("9223372036854775807 >= 9223372036854775807").eval(bindings).longValue() > 0);
Assert.assertTrue(Parser.parse("9223372036854775806 < 9223372036854775807").eval(bindings).longValue() > 0);
Assert.assertTrue(Parser.parse("9223372036854775807 <= 9223372036854775807").eval(bindings).longValue() > 0);
Assert.assertTrue(Parser.parse("9223372036854775807 == 9223372036854775807").eval(bindings).longValue() > 0);
Assert.assertTrue(Parser.parse("9223372036854775807 != 9223372036854775806").eval(bindings).longValue() > 0);
Assert.assertEquals(9223372036854775807L, Parser.parse("9223372036854775806 + 1").eval(bindings).longValue());
Assert.assertEquals(9223372036854775806L, Parser.parse("9223372036854775807 - 1").eval(bindings).longValue());
Assert.assertEquals(9223372036854775806L, Parser.parse("4611686018427387903 * 2").eval(bindings).longValue());
Assert.assertEquals(4611686018427387903L, Parser.parse("9223372036854775806 / 2").eval(bindings).longValue());
Assert.assertEquals(7L, Parser.parse("9223372036854775807 % 9223372036854775800").eval(bindings).longValue());
Assert.assertEquals( 9223372030926249001L, Parser.parse("3037000499 ^ 2").eval(bindings).longValue());
Assert.assertEquals(-9223372036854775807L, Parser.parse("-9223372036854775807").eval(bindings).longValue());
Assert.assertTrue(Parser.parse("!-9223372036854775807").eval(bindings).longValue() > 0);
Assert.assertTrue(Parser.parse("!0").eval(bindings).longValue() > 0);
Assert.assertFalse(Parser.parse("!9223372036854775807").eval(bindings).longValue() > 0);
Assert.assertEquals(3037000499L, Parser.parse("sqrt(9223372036854775807)").eval(bindings).longValue());
Assert.assertEquals(9223372036854775807L, Parser.parse("if(9223372036854775807, 9223372036854775807, 9223372036854775806)").eval(bindings).longValue());
Assert.assertEquals(9223372036854775806L, Parser.parse("if(0, 9223372036854775807, 9223372036854775806)").eval(bindings).longValue());
}
}

266
common/src/test/java/io/druid/math/expr/ParserTest.java Normal file
View File

@ -0,0 +1,266 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.math.expr;
import org.junit.Assert;
import org.junit.Test;
/**
*/
public class ParserTest
{
@Test
public void testSimple()
{
String actual = Parser.parse("1").toString();
String expected = "1";
Assert.assertEquals(expected, actual);
}
@Test
public void testSimpleUnaryOps1()
{
String actual = Parser.parse("-x").toString();
String expected = "-x";
Assert.assertEquals(expected, actual);
actual = Parser.parse("!x").toString();
expected = "!x";
Assert.assertEquals(expected, actual);
}
@Test
public void testSimpleUnaryOps2()
{
String actual = Parser.parse("-1").toString();
String expected = "-1";
Assert.assertEquals(expected, actual);
actual = Parser.parse("--1").toString();
expected = "--1";
Assert.assertEquals(expected, actual);
actual = Parser.parse("-1+2").toString();
expected = "(+ -1 2)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("-1*2").toString();
expected = "(* -1 2)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("-1^2").toString();
expected = "(^ -1 2)";
Assert.assertEquals(expected, actual);
}
@Test
public void testSimpleLogicalOps1()
{
String actual = Parser.parse("x>y").toString();
String expected = "(> x y)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("x<y").toString();
expected = "(< x y)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("x<=y").toString();
expected = "(<= x y)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("x>=y").toString();
expected = "(>= x y)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("x==y").toString();
expected = "(== x y)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("x!=y").toString();
expected = "(!= x y)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("x && y").toString();
expected = "(&& x y)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("x || y").toString();
expected = "(|| x y)";
Assert.assertEquals(expected, actual);
}
@Test
public void testSimpleAdditivityOp1()
{
String actual = Parser.parse("x+y").toString();
String expected = "(+ x y)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("x-y").toString();
expected = "(- x y)";
Assert.assertEquals(expected, actual);
}
@Test
public void testSimpleAdditivityOp2()
{
String actual = Parser.parse("x+y+z").toString();
String expected = "(+ (+ x y) z)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("x+y-z").toString();
expected = "(- (+ x y) z)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("x-y+z").toString();
expected = "(+ (- x y) z)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("x-y-z").toString();
expected = "(- (- x y) z)";
Assert.assertEquals(expected, actual);
}
@Test
public void testSimpleMultiplicativeOp1()
{
String actual = Parser.parse("x*y").toString();
String expected = "(* x y)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("x/y").toString();
expected = "(/ x y)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("x%y").toString();
expected = "(% x y)";
Assert.assertEquals(expected, actual);
}
@Test
public void testSimpleMultiplicativeOp2()
{
String actual = Parser.parse("1*2*3").toString();
String expected = "(* (* 1 2) 3)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("1*2/3").toString();
expected = "(/ (* 1 2) 3)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("1/2*3").toString();
expected = "(* (/ 1 2) 3)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("1/2/3").toString();
expected = "(/ (/ 1 2) 3)";
Assert.assertEquals(expected, actual);
}
@Test
public void testSimpleCarrot1()
{
String actual = Parser.parse("1^2").toString();
String expected = "(^ 1 2)";
Assert.assertEquals(expected, actual);
}
@Test
public void testSimpleCarrot2()
{
String actual = Parser.parse("1^2^3").toString();
String expected = "(^ 1 (^ 2 3))";
Assert.assertEquals(expected, actual);
}
@Test
public void testMixed()
{
String actual = Parser.parse("1+2*3").toString();
String expected = "(+ 1 (* 2 3))";
Assert.assertEquals(expected, actual);
actual = Parser.parse("1+(2*3)").toString();
Assert.assertEquals(expected, actual);
actual = Parser.parse("(1+2)*3").toString();
expected = "(* (+ 1 2) 3)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("1*2+3").toString();
expected = "(+ (* 1 2) 3)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("(1*2)+3").toString();
Assert.assertEquals(expected, actual);
actual = Parser.parse("1*(2+3)").toString();
expected = "(* 1 (+ 2 3))";
Assert.assertEquals(expected, actual);
actual = Parser.parse("1+2^3").toString();
expected = "(+ 1 (^ 2 3))";
Assert.assertEquals(expected, actual);
actual = Parser.parse("1+(2^3)").toString();
expected = "(+ 1 (^ 2 3))";
Assert.assertEquals(expected, actual);
actual = Parser.parse("(1+2)^3").toString();
expected = "(^ (+ 1 2) 3)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("1^2+3").toString();
expected = "(+ (^ 1 2) 3)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("(1^2)+3").toString();
expected = "(+ (^ 1 2) 3)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("1^(2+3)").toString();
expected = "(^ 1 (+ 2 3))";
Assert.assertEquals(expected, actual);
actual = Parser.parse("1^2*3+4").toString();
expected = "(+ (* (^ 1 2) 3) 4)";
Assert.assertEquals(expected, actual);
actual = Parser.parse("-1^-2*-3+-4").toString();
expected = "(+ (* (^ -1 -2) -3) -4)";
Assert.assertEquals(expected, actual);
}
@Test
public void testFunctions()
{
String actual = Parser.parse("sqrt(x)").toString();
String expected = "(sqrt [x])";
Assert.assertEquals(expected, actual);
actual = Parser.parse("if(cond,then,else)").toString();
expected = "(if [cond, then, else])";
Assert.assertEquals(expected, actual);
}
}

28
docs/content/misc/math-expr.md Normal file
View File

@ -0,0 +1,28 @@
---
layout: doc_page
---
This expression language supports the following operators (listed in decreasing order of precedence).
|Operators|Description|
|---------|-----------|
|!, -|Unary NOT and Minus|
|^|Binary power op|
|*, /, %|Binary multiplicative|
|+, -|Binary additive|
|<, <=, >, >=, ==, !=|Binary Comparison|
|&&,\|\||Binary Logical AND, OR|
Long and double data types are supported. If a number contains a dot, it is interpreted as a double, otherwise it is interpreted as a long. That means, always add a '.' to your number if you want it intepreted as a double value.
Expressions can contain variables. Variable names may contain letters, digits, '\_' and '$'. Variable names must not begin with a digit.
For logical operators, a number is true if and only if it is positive. (0 means false)
Also, the following in-built functions are supported.
|name|description|
|----|-----------|
|sqrt|sqrt(x) would return square root of x|
|if|if(predicate,then,else) returns 'then' if 'predicate' evaluates to a positive number, otherwise it returns 'else'|

6
pom.xml
View File

@ -444,12 +444,12 @@
<dependency> <dependency>
<groupId>org.antlr</groupId> <groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId> <artifactId>antlr4-runtime</artifactId>
<version>4.0</version> <version>4.5.1</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.antlr</groupId> <groupId>org.antlr</groupId>
<artifactId>antlr4-coordinator</artifactId> <artifactId>antlr4-coordinator</artifactId>
<version>4.0</version> <version>4.5.1</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>commons-cli</groupId> <groupId>commons-cli</groupId>
@ -740,7 +740,7 @@
<plugin> <plugin>
<groupId>org.antlr</groupId> <groupId>org.antlr</groupId>
<artifactId>antlr4-maven-plugin</artifactId> <artifactId>antlr4-maven-plugin</artifactId>
<version>4.0</version> <version>4.5.1</version>
</plugin> </plugin>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>

15
server/pom.xml
View File

@ -132,10 +132,6 @@
<groupId>org.eclipse.aether</groupId> <groupId>org.eclipse.aether</groupId>
<artifactId>aether-api</artifactId> <artifactId>aether-api</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
</dependency>
<dependency> <dependency>
<groupId>net.spy</groupId> <groupId>net.spy</groupId>
<artifactId>spymemcached</artifactId> <artifactId>spymemcached</artifactId>
@ -222,17 +218,6 @@
</execution> </execution>
</executions> </executions>
</plugin> </plugin>
<plugin>
<groupId>org.antlr</groupId>
<artifactId>antlr4-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>antlr4</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins> </plugins>
</build> </build>

343
server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4
View File

@ -1,343 +0,0 @@
grammar DruidSQL;
@header {
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import io.druid.granularity.PeriodGranularity;
import io.druid.granularity.QueryGranularity;
import io.druid.query.aggregation.AggregatorFactory;
import io.druid.query.aggregation.CountAggregatorFactory;
import io.druid.query.aggregation.DoubleSumAggregatorFactory;
import io.druid.query.aggregation.DoubleMaxAggregatorFactory;
import io.druid.query.aggregation.DoubleMinAggregatorFactory;
import io.druid.query.aggregation.PostAggregator;
import io.druid.query.aggregation.post.ArithmeticPostAggregator;
import io.druid.query.aggregation.post.ConstantPostAggregator;
import io.druid.query.aggregation.post.FieldAccessPostAggregator;
import io.druid.query.dimension.DefaultDimensionSpec;
import io.druid.query.dimension.DimensionSpec;
import io.druid.query.filter.AndDimFilter;
import io.druid.query.filter.DimFilter;
import io.druid.query.filter.NotDimFilter;
import io.druid.query.filter.OrDimFilter;
import io.druid.query.filter.RegexDimFilter;
import io.druid.query.filter.SelectorDimFilter;
import org.antlr.v4.runtime.NoViableAltException;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNSimulator;
import org.antlr.v4.runtime.atn.ParserATNSimulator;
import org.antlr.v4.runtime.atn.PredictionContextCache;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.tree.ParseTreeListener;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.joda.time.DateTime;
import org.joda.time.Period;
import java.text.NumberFormat;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
}
@parser::members {
public Map<String, AggregatorFactory> aggregators = new LinkedHashMap<String, AggregatorFactory>();
public List<PostAggregator> postAggregators = new LinkedList<PostAggregator>();
public DimFilter filter;
public List<org.joda.time.Interval> intervals;
public List<String> fields = new LinkedList<String>();
public QueryGranularity granularity = QueryGranularity.ALL;
public Map<String, DimensionSpec> groupByDimensions = new LinkedHashMap<String, DimensionSpec>();
String dataSourceName = null;
public String getDataSource() {
return dataSourceName;
}
public String unescape(String quoted) {
String unquote = quoted.trim().replaceFirst("^'(.*)'\$", "\$1");
return unquote.replace("''", "'");
}
AggregatorFactory evalAgg(String name, int fn) {
switch (fn) {
case SUM: return new DoubleSumAggregatorFactory("sum("+name+")", name);
case MIN: return new DoubleMinAggregatorFactory("min("+name+")", name);
case MAX: return new DoubleMaxAggregatorFactory("max("+name+")", name);
case COUNT: return new CountAggregatorFactory(name);
}
throw new IllegalArgumentException("Unknown function [" + fn + "]");
}
PostAggregator evalArithmeticPostAggregator(PostAggregator a, List<Token> ops, List<PostAggregator> b) {
if(b.isEmpty()) return a;
else {
int i = 0;
PostAggregator root = a;
while(i < ops.size()) {
List<PostAggregator> list = new LinkedList<PostAggregator>();
List<String> names = new LinkedList<String>();
names.add(root.getName());
list.add(root);
Token op = ops.get(i);
while(i < ops.size() && ops.get(i).getType() == op.getType()) {
PostAggregator e = b.get(i);
list.add(e);
names.add(e.getName());
i++;
}
root = new ArithmeticPostAggregator("("+Joiner.on(op.getText()).join(names)+")", op.getText(), list);
}
return root;
}
}
}
AND: 'and';
OR: 'or';
SUM: 'sum';
MIN: 'min';
MAX: 'max';
COUNT: 'count';
AS: 'as';
OPEN: '(';
CLOSE: ')';
STAR: '*';
NOT: '!' ;
PLUS: '+';
MINUS: '-';
DIV: '/';
COMMA: ',';
EQ: '=';
NEQ: '!=';
MATCH: '~';
GROUP: 'group';
IDENT : (LETTER)(LETTER | DIGIT | '_')* ;
QUOTED_STRING : '\'' ( ESC | ~'\'' )*? '\'' ;
ESC : '\'' '\'';
NUMBER: DIGIT*'.'?DIGIT+(EXPONENT)?;
EXPONENT: ('e') ('+'|'-')? ('0'..'9')+;
fragment DIGIT : '0'..'9';
fragment LETTER : 'a'..'z' | 'A'..'Z';
LINE_COMMENT : '--' .*? '\r'? '\n' -> skip ;
COMMENT : '/*' .*? '*/' -> skip ;
WS : (' '| '\t' | '\r' '\n' | '\n' | '\r')+ -> skip;
query
: select_stmt where_stmt (groupby_stmt)?
;
select_stmt
: 'select' e+=aliasedExpression (',' e+=aliasedExpression)* 'from' datasource {
for(AliasedExpressionContext a : $e) {
postAggregators.add(a.p);
fields.add(a.p.getName());
}
this.dataSourceName = $datasource.text;
}
;
where_stmt
: 'where' f=timeAndDimFilter {
if($f.filter != null) this.filter = $f.filter;
this.intervals = Lists.newArrayList($f.interval);
}
;
groupby_stmt
: GROUP 'by' groupByExpression ( COMMA! groupByExpression )*
;
groupByExpression
: gran=granularityFn {this.granularity = $gran.granularity;}
| dim=IDENT { this.groupByDimensions.put($dim.text, new DefaultDimensionSpec($dim.text, $dim.text)); }
;
datasource
: IDENT
;
aliasedExpression returns [PostAggregator p]
: expression ( AS^ name=IDENT )? {
if($name != null) {
postAggregators.add($expression.p);
$p = new FieldAccessPostAggregator($name.text, $expression.p.getName());
}
else $p = $expression.p;
}
;
expression returns [PostAggregator p]
: additiveExpression { $p = $additiveExpression.p; }
;
additiveExpression returns [PostAggregator p]
: a=multiplyExpression (( ops+=PLUS^ | ops+=MINUS^ ) b+=multiplyExpression)* {
List<PostAggregator> rhs = new LinkedList<PostAggregator>();
for(MultiplyExpressionContext e : $b) rhs.add(e.p);
$p = evalArithmeticPostAggregator($a.p, $ops, rhs);
}
;
multiplyExpression returns [PostAggregator p]
: a=unaryExpression ((ops+= STAR | ops+=DIV ) b+=unaryExpression)* {
List<PostAggregator> rhs = new LinkedList<PostAggregator>();
for(UnaryExpressionContext e : $b) rhs.add(e.p);
$p = evalArithmeticPostAggregator($a.p, $ops, rhs);
}
;
unaryExpression returns [PostAggregator p]
: MINUS e=unaryExpression {
if($e.p instanceof ConstantPostAggregator) {
ConstantPostAggregator c = (ConstantPostAggregator)$e.p;
double v = c.getConstantValue().doubleValue() * -1;
$p = new ConstantPostAggregator(Double.toString(v), v);
} else {
$p = new ArithmeticPostAggregator(
"-"+$e.p.getName(),
"*",
Lists.newArrayList($e.p, new ConstantPostAggregator("-1", -1.0))
);
}
}
| PLUS e=unaryExpression { $p = $e.p; }
| primaryExpression { $p = $primaryExpression.p; }
;
primaryExpression returns [PostAggregator p]
: constant { $p = $constant.c; }
| aggregate {
aggregators.put($aggregate.agg.getName(), $aggregate.agg);
$p = new FieldAccessPostAggregator($aggregate.agg.getName(), $aggregate.agg.getName());
}
| OPEN! e=expression CLOSE! { $p = $e.p; }
;
aggregate returns [AggregatorFactory agg]
: fn=( SUM^ | MIN^ | MAX^ ) OPEN! name=(IDENT|COUNT) CLOSE! { $agg = evalAgg($name.text, $fn.type); }
| fn=COUNT OPEN! STAR CLOSE! { $agg = evalAgg("count(*)", $fn.type); }
;
constant returns [ConstantPostAggregator c]
: value=NUMBER { double v = Double.parseDouble($value.text); $c = new ConstantPostAggregator(Double.toString(v), v); }
;
/* time filters must be top level filters */
timeAndDimFilter returns [DimFilter filter, org.joda.time.Interval interval]
: (f1=dimFilter AND)? t=timeFilter (AND f2=dimFilter)? {
if($f1.ctx != null || $f2.ctx != null) {
if($f1.ctx != null && $f2.ctx != null) {
$filter = new AndDimFilter(Lists.newArrayList($f1.filter, $f2.filter));
} else if($f1.ctx != null) {
$filter = $f1.filter;
} else {
$filter = $f2.filter;
}
}
$interval = $t.interval;
}
;
dimFilter returns [DimFilter filter]
: e=orDimFilter { $filter = $e.filter; }
;
orDimFilter returns [DimFilter filter]
: a=andDimFilter (OR^ b+=andDimFilter)* {
if($b.isEmpty()) $filter = $a.filter;
else {
List<DimFilter> rest = new ArrayList<DimFilter>();
for(AndDimFilterContext e : $b) rest.add(e.filter);
$filter = new OrDimFilter(Lists.asList($a.filter, rest.toArray(new DimFilter[]{})));
}
}
;
andDimFilter returns [DimFilter filter]
: a=primaryDimFilter (AND^ b+=primaryDimFilter)* {
if($b.isEmpty()) $filter = $a.filter;
else {
List<DimFilter> rest = new ArrayList<DimFilter>();
for(PrimaryDimFilterContext e : $b) rest.add(e.filter);
$filter = new AndDimFilter(Lists.asList($a.filter, rest.toArray(new DimFilter[]{})));
}
}
;
primaryDimFilter returns [DimFilter filter]
: e=selectorDimFilter { $filter = $e.filter; }
| l=inListDimFilter { $filter = $l.filter; }
| NOT f=dimFilter { $filter = new NotDimFilter($f.filter); }
| OPEN! f=dimFilter CLOSE! { $filter = $f.filter; }
;
selectorDimFilter returns [DimFilter filter]
: dimension=IDENT op=(EQ|NEQ|MATCH) value=QUOTED_STRING {
String dim = $dimension.text;
String val = unescape($value.text);
switch($op.type) {
case(EQ): $filter = new SelectorDimFilter(dim, val, null); break;
case(NEQ): $filter = new NotDimFilter(new SelectorDimFilter(dim, val, null)); break;
case(MATCH): $filter = new RegexDimFilter(dim, val, null); break;
}
}
;
inListDimFilter returns [DimFilter filter]
: dimension=IDENT 'in' (OPEN! ( (list+=QUOTED_STRING (COMMA! list+=QUOTED_STRING)*) ) CLOSE!) {
List<DimFilter> filterList = new LinkedList<DimFilter>();
for(Token e : $list) filterList.add(new SelectorDimFilter($dimension.text, unescape(e.getText()), null));
$filter = new OrDimFilter(filterList);
}
;
timeFilter returns [org.joda.time.Interval interval, QueryGranularity granularity]
: 'timestamp' 'between' s=timestamp AND e=timestamp {
$interval = new org.joda.time.Interval($s.t, $e.t);
}
;
granularityFn returns [QueryGranularity granularity]
: 'granularity' OPEN! 'timestamp' ',' str=QUOTED_STRING CLOSE! {
String granStr = unescape($str.text);
try {
$granularity = QueryGranularity.fromString(granStr);
} catch(IllegalArgumentException e) {
$granularity = new PeriodGranularity(new Period(granStr), null, null);
}
}
;
timestamp returns [DateTime t]
: NUMBER {
String str = $NUMBER.text.trim();
try {
$t = new DateTime(NumberFormat.getInstance().parse(str));
}
catch(ParseException e) {
throw new IllegalArgumentException("Unable to parse number [" + str + "]");
}
}
| QUOTED_STRING { $t = new DateTime(unescape($QUOTED_STRING.text)); }
;

229
server/src/main/java/io/druid/server/sql/SQLRunner.java
View File

@ -1,229 +0,0 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.server.sql;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.base.Charsets;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.metamx.common.StringUtils;
import com.metamx.common.guava.CloseQuietly;
import io.druid.data.input.Row;
import io.druid.jackson.DefaultObjectMapper;
import io.druid.query.Druids;
import io.druid.query.Query;
import io.druid.query.Result;
import io.druid.query.aggregation.AggregatorFactory;
import io.druid.query.dimension.DimensionSpec;
import io.druid.query.groupby.GroupByQuery;
import io.druid.query.timeseries.TimeseriesResultValue;
import io.druid.sql.antlr4.DruidSQLLexer;
import io.druid.sql.antlr4.DruidSQLParser;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.ConsoleErrorListener;
import org.antlr.v4.runtime.TokenStream;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import javax.annotation.Nullable;
import javax.ws.rs.core.MediaType;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
public class SQLRunner
{
private static final String STATEMENT = "select count(*), (1 - count(*) / sum(count)) * 100 as ratio from wikipedia where"
+ " timestamp between '2013-02-01' and '2013-02-14'"
+ " and (namespace = 'article' or page ~ 'Talk:.*')"
+ " and language in ( 'en', 'fr' ) "
+ " and user ~ '(?i)^david.*'"
+ " group by granularity(timestamp, 'day'), language";
public static void main(String[] args) throws Exception
{
Options options = new Options();
options.addOption("h", "help", false, "help");
options.addOption("v", false, "verbose");
options.addOption("e", "host", true, "endpoint [hostname:port]");
CommandLine cmd = new GnuParser().parse(options, args);
if(cmd.hasOption("h")) {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("SQLRunner", options);
System.exit(2);
}
String hostname = cmd.getOptionValue("e", "localhost:8080");
String sql = cmd.getArgs().length > 0 ? cmd.getArgs()[0] : STATEMENT;
ObjectMapper objectMapper = new DefaultObjectMapper();
ObjectWriter jsonWriter = objectMapper.writerWithDefaultPrettyPrinter();
CharStream stream = new ANTLRInputStream(sql);
DruidSQLLexer lexer = new DruidSQLLexer(stream);
TokenStream tokenStream = new CommonTokenStream(lexer);
DruidSQLParser parser = new DruidSQLParser(tokenStream);
lexer.removeErrorListeners();
parser.removeErrorListeners();
lexer.addErrorListener(ConsoleErrorListener.INSTANCE);
parser.addErrorListener(ConsoleErrorListener.INSTANCE);
try {
DruidSQLParser.QueryContext queryContext = parser.query();
if(parser.getNumberOfSyntaxErrors() > 0) throw new IllegalStateException();
// parser.setBuildParseTree(true);
// System.err.println(q.toStringTree(parser));
} catch(Exception e) {
String msg = e.getMessage();
if(msg != null) System.err.println(e);
System.exit(1);
}
final Query query;
final TypeReference typeRef;
boolean groupBy = false;
if(parser.groupByDimensions.isEmpty()) {
query = Druids.newTimeseriesQueryBuilder()
.dataSource(parser.getDataSource())
.aggregators(new ArrayList<AggregatorFactory>(parser.aggregators.values()))
.postAggregators(parser.postAggregators)
.intervals(parser.intervals)
.granularity(parser.granularity)
.filters(parser.filter)
.build();
typeRef = new TypeReference<List<Result<TimeseriesResultValue>>>(){};
} else {
query = GroupByQuery.builder()
.setDataSource(parser.getDataSource())
.setAggregatorSpecs(new ArrayList<AggregatorFactory>(parser.aggregators.values()))
.setPostAggregatorSpecs(parser.postAggregators)
.setInterval(parser.intervals)
.setGranularity(parser.granularity)
.setDimFilter(parser.filter)
.setDimensions(new ArrayList<DimensionSpec>(parser.groupByDimensions.values()))
.build();
typeRef = new TypeReference<List<Row>>(){};
groupBy = true;
}
String queryStr = jsonWriter.writeValueAsString(query);
if(cmd.hasOption("v")) System.err.println(queryStr);
URL url = new URL(String.format("http://%s/druid/v2/?pretty", hostname));
final URLConnection urlConnection = url.openConnection();
urlConnection.addRequestProperty("content-type", MediaType.APPLICATION_JSON);
urlConnection.getOutputStream().write(StringUtils.toUtf8(queryStr));
BufferedReader stdInput = new BufferedReader(new InputStreamReader(urlConnection.getInputStream(), Charsets.UTF_8));
Object res = objectMapper.readValue(stdInput, typeRef);
Joiner tabJoiner = Joiner.on("\t");
if(groupBy) {
List<Row> rows = (List<Row>)res;
Iterable<String> dimensions = Iterables.transform(parser.groupByDimensions.values(), new Function<DimensionSpec, String>()
{
@Override
public String apply(@Nullable DimensionSpec input)
{
return input.getOutputName();
}
});
System.out.println(tabJoiner.join(Iterables.concat(
Lists.newArrayList("timestamp"),
dimensions,
parser.fields
)));
for(final Row r : rows) {
System.out.println(
tabJoiner.join(
Iterables.concat(
Lists.newArrayList(parser.granularity.toDateTime(r.getTimestampFromEpoch())),
Iterables.transform(
parser.groupByDimensions.values(), new Function<DimensionSpec, String>()
{
@Override
public String apply(@Nullable DimensionSpec input)
{
return Joiner.on(",").join(r.getDimension(input.getOutputName()));
}
}),
Iterables.transform(parser.fields, new Function<String, Object>()
{
@Override
public Object apply(@Nullable String input)
{
return r.getFloatMetric(input);
}
})
)
)
);
}
}
else {
List<Result<TimeseriesResultValue>> rows = (List<Result<TimeseriesResultValue>>)res;
System.out.println(tabJoiner.join(Iterables.concat(
Lists.newArrayList("timestamp"),
parser.fields
)));
for(final Result<TimeseriesResultValue> r : rows) {
System.out.println(
tabJoiner.join(
Iterables.concat(
Lists.newArrayList(r.getTimestamp()),
Lists.transform(
parser.fields,
new Function<String, Object>()
{
@Override
public Object apply(@Nullable String input)
{
return r.getValue().getMetric(input);
}
}
)
)
)
);
}
}
CloseQuietly.close(stdInput);
}
}