From 36ccfbd20e2e73889de8d86a4a5b31a1b0ed22b7 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Mon, 14 Dec 2015 01:15:50 -0600 Subject: [PATCH 1/3] math expression language with hand written parser/lexer --- .../main/java/io/druid/math/expr/Expr.java | 398 ++++++++++++++++++ .../java/io/druid/math/expr/Function.java | 64 +++ .../main/java/io/druid/math/expr/Lexer.java | 229 ++++++++++ .../main/java/io/druid/math/expr/Parser.java | 127 ++++++ .../main/java/io/druid/math/expr/Token.java | 180 ++++++++ .../java/io/druid/math/expr/EvalTest.java | 109 +++++ .../java/io/druid/math/expr/LexerTest.java | 61 +++ .../java/io/druid/math/expr/ParserTest.java | 266 ++++++++++++ docs/content/misc/math-expr.md | 34 ++ 9 files changed, 1468 insertions(+) create mode 100644 common/src/main/java/io/druid/math/expr/Expr.java create mode 100644 common/src/main/java/io/druid/math/expr/Function.java create mode 100644 common/src/main/java/io/druid/math/expr/Lexer.java create mode 100644 common/src/main/java/io/druid/math/expr/Parser.java create mode 100644 common/src/main/java/io/druid/math/expr/Token.java create mode 100644 common/src/test/java/io/druid/math/expr/EvalTest.java create mode 100644 common/src/test/java/io/druid/math/expr/LexerTest.java create mode 100644 common/src/test/java/io/druid/math/expr/ParserTest.java create mode 100644 docs/content/misc/math-expr.md diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java new file mode 100644 index 00000000000..7e24c11d25f --- /dev/null +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -0,0 +1,398 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +import com.google.common.math.LongMath; + +import java.util.List; +import java.util.Map; + +/** + */ +public interface Expr +{ + Number eval(Map bindings); +} + +class SimpleExpr implements Expr +{ + private final Atom atom; + + public SimpleExpr(Atom atom) + { + this.atom = atom; + } + + @Override + public String toString() + { + return atom.toString(); + } + + @Override + public Number eval(Map bindings) + { + return atom.eval(bindings); + } +} + +class BinExpr implements Expr +{ + private final Token opToken; + private final Expr left; + private final Expr right; + + public BinExpr(Token opToken, Expr left, Expr right) + { + this.opToken = opToken; + this.left = left; + this.right = right; + } + + public Number eval(Map bindings) + { + switch(opToken.getType()) { + case Token.AND: + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + long lval = leftVal.longValue(); + if (lval > 0) { + long rval = rightVal.longValue(); + return rval > 0 ? 1 : 0; + } else { + return 0; + } + } else { + double lval = leftVal.doubleValue(); + if (lval > 0) { + double rval = rightVal.doubleValue(); + return rval > 0 ? 1.0d : 0.0d; + } else { + return 0.0d; + } + } + case Token.OR: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + long lval = leftVal.longValue(); + if (lval > 0) { + return 1; + } else { + long rval = rightVal.longValue(); + return rval > 0 ? 1 : 0; + } + } else { + double lval = leftVal.doubleValue(); + if (lval > 0) { + return 1.0d; + } else { + double rval = rightVal.doubleValue(); + return rval > 0 ? 1.0d : 0.0d; + } + } + case Token.LT: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() < rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() < rightVal.doubleValue() ? 1.0d : 0.0d; + } + case Token.LEQ: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() <= rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() <= rightVal.doubleValue() ? 1.0d : 0.0d; + } + case Token.GT: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() > rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() > rightVal.doubleValue() ? 1.0d : 0.0d; + } + case Token.GEQ: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() >= rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() >= rightVal.doubleValue() ? 1.0d : 0.0d; + } + case Token.EQ: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() == rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() == rightVal.doubleValue() ? 1.0d : 0.0d; + } + case Token.NEQ: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() != rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() != rightVal.doubleValue() ? 1.0d : 0.0d; + } + case Token.PLUS: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() + rightVal.longValue(); + } else { + return leftVal.doubleValue() + rightVal.doubleValue(); + } + case Token.MINUS: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() - rightVal.longValue(); + } else { + return leftVal.doubleValue() - rightVal.doubleValue(); + } + case Token.MUL: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() * rightVal.longValue(); + } else { + return leftVal.doubleValue() * rightVal.doubleValue(); + } + case Token.DIV: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() / rightVal.longValue(); + } else { + return leftVal.doubleValue() / rightVal.doubleValue(); + } + case Token.MODULO: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() % rightVal.longValue(); + } else { + return leftVal.doubleValue() % rightVal.doubleValue(); + } + case Token.CARROT: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return LongMath.pow(leftVal.longValue(), rightVal.intValue()); + } else { + return Math.pow(leftVal.doubleValue(), rightVal.doubleValue()); + } + default: + throw new RuntimeException("Unknown operator " + opToken.getMatch()); + } + } + + private boolean isLong(Number left, Number right) + { + return left instanceof Long && right instanceof Long; + } + + @Override + public String toString() + { + return "(" + opToken.getMatch() + " " + left + " " + right + ")"; + } +} + + + +interface Atom +{ + Number eval(Map bindings); +} + +class LongValueAtom implements Atom +{ + private final long value; + + public LongValueAtom(long value) + { + this.value = value; + } + + @Override + public String toString() + { + return String.valueOf(value); + } + + @Override + public Number eval(Map bindings) + { + return value; + } +} + +class DoubleValueAtom implements Atom +{ + private final double value; + + public DoubleValueAtom(double value) + { + this.value = value; + } + + @Override + public String toString() + { + return String.valueOf(value); + } + + @Override + public Number eval(Map bindings) + { + return value; + } +} + +class IdentifierAtom implements Atom +{ + private final Token value; + + public IdentifierAtom(Token value) + { + this.value = value; + } + + @Override + public String toString() + { + return value.getMatch(); + } + + @Override + public Number eval(Map bindings) + { + Number val = bindings.get(value.getMatch()); + if (val == null) { + throw new RuntimeException("No binding found for " + value.getMatch()); + } else { + return val; + } + } +} + +class UnaryNotExprAtom implements Atom +{ + private final Expr expr; + + public UnaryNotExprAtom(Expr expr) + { + this.expr = expr; + } + + @Override + public String toString() + { + return "!" + expr.toString(); + } + + @Override + public Number eval(Map bindings) + { + Number valObj = expr.eval(bindings); + return valObj.doubleValue() > 0 ? 0.0d : 1.0d; + } +} + +class UnaryMinusExprAtom implements Atom +{ + private final Expr expr; + + public UnaryMinusExprAtom(Expr expr) + { + this.expr = expr; + } + + @Override + public String toString() + { + return "-" + expr.toString(); + } + + @Override + public Number eval(Map bindings) + { + Number valObj = expr.eval(bindings); + if (valObj instanceof Long) { + return -1 * valObj.longValue(); + } else { + return -1 * valObj.doubleValue(); + } + } +} + + +class NestedExprAtom implements Atom +{ + private final Expr expr; + + public NestedExprAtom(Expr expr) + { + this.expr = expr; + } + + @Override + public String toString() + { + return expr.toString(); + } + + @Override + public Number eval(Map bindings) + { + return expr.eval(bindings); + } +} + +class FunctionAtom implements Atom +{ + private final String name; + private final List args; + + public FunctionAtom(String name, List args) + { + this.name = name; + this.args = args; + } + + @Override + public String toString() + { + return "(" + name + " " + args + ")"; + } + + @Override + public Number eval(Map bindings) + { + return Parser.func.get(name).apply(args, bindings); + } +} diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java new file mode 100644 index 00000000000..630642ba867 --- /dev/null +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -0,0 +1,64 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +import java.util.List; +import java.util.Map; + +/** + */ +interface Function +{ + Number apply(List args, Map bindings); +} + +class SqrtFunc implements Function +{ + + @Override + public Number apply(List args, Map bindings) + { + if (args.size() != 1) { + throw new RuntimeException("function 'sqrt' needs 1 argument"); + } + + Number x = args.get(0).eval(bindings); + return Math.sqrt(x.doubleValue()); + } +} + +class ConditionFunc implements Function +{ + + @Override + public Number apply(List args, Map bindings) + { + if (args.size() != 3) { + throw new RuntimeException("function 'if' needs 3 argument"); + } + + Number x = args.get(0).eval(bindings); + if (x instanceof Long) { + return x.longValue() > 0 ? args.get(1).eval(bindings) : args.get(2).eval(bindings); + } else { + return x.doubleValue() > 0 ? args.get(1).eval(bindings) : args.get(2).eval(bindings); + } + } +} diff --git a/common/src/main/java/io/druid/math/expr/Lexer.java b/common/src/main/java/io/druid/math/expr/Lexer.java new file mode 100644 index 00000000000..fc55a689ea6 --- /dev/null +++ b/common/src/main/java/io/druid/math/expr/Lexer.java @@ -0,0 +1,229 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +class Lexer +{ + private String input; + private int currPos; + + private Token next; + + Lexer(String input) + { + this.input = input; + currPos = 0; + + next = nextToken(); + } + + Token peek() + { + return next; + } + + Token consume() + { + Token old = next; + next = nextToken(); + return old; + } + + private Token nextToken() + { + if (currPos >= input.length()) { + return new Token(Token.EOF); + } + + char c = input.charAt(currPos); + while(c == ' ') { + currPos++; + if (currPos >= input.length()) { + return new Token(Token.EOF); + } + c = input.charAt(currPos); + } + + switch(c) + { + case '<': + currPos++; + if (currPos < input.length() && input.charAt(currPos) == '=') { + currPos++; + return new Token(Token.LEQ, "<="); + } else { + return new Token(Token.LT, "<"); + } + case '>': + currPos++; + if (currPos < input.length() && input.charAt(currPos) == '=') { + currPos++; + return new Token(Token.GEQ, ">="); + } else { + return new Token(Token.GT, ">"); + } + case '=': + currPos++; + if (currPos < input.length() && input.charAt(currPos) == '=') { + currPos++; + return new Token(Token.EQ, "=="); + } else { + throw new IllegalArgumentException("unknown operator '='"); + } + case '!': + currPos++; + if (currPos < input.length() && input.charAt(currPos) == '=') { + currPos++; + return new Token(Token.NEQ, "!="); + } else { + return new Token(Token.NOT, "!"); + } + + case '+': + currPos++; + return new Token(Token.PLUS, "+"); + case '-': + currPos++; + return new Token(Token.MINUS, "-"); + + case '*': + currPos++; + return new Token(Token.MUL, "*"); + case '/': + currPos++; + return new Token(Token.DIV, "/"); + case '%': + currPos++; + return new Token(Token.MODULO, "%"); + + case '^': + currPos++; + return new Token(Token.CARROT, "^"); + + case '(': + currPos++; + return new Token(Token.LPAREN, "("); + case ')': + currPos++; + return new Token(Token.RPAREN, ")"); + case ',': + currPos++; + return new Token(Token.COMMA, ","); + + default: + if (isNumberStartingChar(c)) { + return parseNumber(); + } else if (isIdentifierStartingChar(c)){ + return parseIdentifierOrKeyword(); + } else { + throw new RuntimeException("Illegal expression " + toString()); + } + } + } + + private boolean isNumberStartingChar(char c) + { + return c >= '0' && c <= '9'; + } + + private boolean isIdentifierStartingChar(char c) + { + return (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + c == '$' || + c == '_'; + } + + private boolean isIdentifierChar(char c) + { + return (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '$' || + c == '_'; + } + + private Token parseIdentifierOrKeyword() + { + StringBuilder sb = new StringBuilder(); + char c = input.charAt(currPos++); + while (isIdentifierChar(c)) { + sb.append(c); + + if (currPos < input.length()) { + c = input.charAt(currPos++); + } else { + currPos++; + break; + } + }; + + currPos--; + + String str = sb.toString(); + if(str.equals("and")) { + return new Token(Token.AND, str); + } else if(str.equals("or")) { + return new Token(Token.OR, str); + } else { + return new Token(Token.IDENTIFIER, sb.toString()); + } + } + + // Numbers + // long : [0-9]+ + // double : [0-9]+.[0-9]+ + private Token parseNumber() + { + boolean isLong = true; + StringBuilder sb = new StringBuilder(); + char c = input.charAt(currPos++); + + while ( + ('0' <= c && c <= '9') || c == '.' + ) { + if (c == '.') { + isLong = false; + } + + sb.append(c); + + if (currPos < input.length()) { + c = input.charAt(currPos++); + } else { + currPos++; + break; + } + }; + + currPos--; + if (isLong) { + return new Token(Token.LONG, sb.toString()); + } else { + return new Token(Token.DOUBLE, sb.toString()); + } + } + + @Override + public String toString() + { + return "at " + currPos + ", " + input; + } +} diff --git a/common/src/main/java/io/druid/math/expr/Parser.java b/common/src/main/java/io/druid/math/expr/Parser.java new file mode 100644 index 00000000000..588db098f45 --- /dev/null +++ b/common/src/main/java/io/druid/math/expr/Parser.java @@ -0,0 +1,127 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Parser +{ + static final Map func = new HashMap<>(); + + static { + func.put("sqrt", new SqrtFunc()); + func.put("if", new ConditionFunc()); + } + + private final Lexer lexer; + + private Parser(Lexer lexer) + { + this.lexer = lexer; + } + + public static Expr parse(String input) + { + return new Parser(new Lexer(input)).parseExpr(0); + } + + private Expr parseExpr(int p) + { + Expr result = new SimpleExpr(parseAtom()); + + Token t = lexer.peek(); + while ( + t.getType() < Token.NUM_BINARY_OPERATORS + && + Token.PRECEDENCE[t.getType()] >= p + ) { + lexer.consume(); + result = new BinExpr( + t, + result, + parseExpr(Token.R_PRECEDENCE[t.getType()]) + ); + + t = lexer.peek(); + } + + return result; + } + + private Atom parseAtom() + { + Token t = lexer.peek(); + + switch(t.getType()) { + case Token.IDENTIFIER: + lexer.consume(); + String id = t.getMatch(); + + if (func.containsKey(id)) { + expect(Token.LPAREN); + List args = new ArrayList<>(); + t = lexer.peek(); + while(t.getType() != Token.RPAREN) { + args.add(parseExpr(0)); + t = lexer.peek(); + if (t.getType() == Token.COMMA) { + lexer.consume(); + } + } + expect(Token.RPAREN); + return new FunctionAtom(id, args); + } + + return new IdentifierAtom(t); + case Token.LONG: + lexer.consume(); + return new LongValueAtom(Long.valueOf(t.getMatch())); + case Token.DOUBLE: + lexer.consume(); + return new DoubleValueAtom(Double.valueOf(t.getMatch())); + case Token.MINUS: + lexer.consume(); + return new UnaryMinusExprAtom(parseExpr(Token.UNARY_MINUS_PRECEDENCE)); + case Token.NOT: + lexer.consume(); + return new UnaryNotExprAtom(parseExpr(Token.UNARY_NOT_PRECEDENCE)); + case Token.LPAREN: + lexer.consume(); + Expr expression = parseExpr(0); + if(lexer.consume().getType() == Token.RPAREN) { + return new NestedExprAtom(expression); + } + default: + throw new RuntimeException("Invalid token found " + t + " in input " + lexer); + } + } + + private void expect(int type) + { + Token t = lexer.consume(); + if(t.getType() != type) { + throw new RuntimeException("Invalid token found " + t + " in input " + lexer); + } + } +} diff --git a/common/src/main/java/io/druid/math/expr/Token.java b/common/src/main/java/io/druid/math/expr/Token.java new file mode 100644 index 00000000000..25cf6ac3e34 --- /dev/null +++ b/common/src/main/java/io/druid/math/expr/Token.java @@ -0,0 +1,180 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +/** + */ +class Token +{ + static final int AND = 0; + static final int OR = 1; + + static final int LT = 2; + static final int LEQ = 3; + static final int GT = 4; + static final int GEQ = 5; + static final int EQ = 6; + static final int NEQ = 7; + + static final int PLUS = 8; + static final int MINUS = 9; + + static final int MUL = 10; + static final int DIV = 11; + static final int MODULO = 12; + + static final int CARROT = 13; + static final int NUM_BINARY_OPERATORS = 14; + + static final int LPAREN = 51; + static final int RPAREN = 52; + static final int COMMA = 53; + static final int NOT = 54; + static final int IDENTIFIER = 55; + static final int LONG = 56; + static final int DOUBLE = 57; + + static final int EOF = 100; + + static final int[] PRECEDENCE; + static final int[] R_PRECEDENCE; + static final int UNARY_MINUS_PRECEDENCE; + static final int UNARY_NOT_PRECEDENCE; + + static { + PRECEDENCE = new int[NUM_BINARY_OPERATORS]; + R_PRECEDENCE = new int[NUM_BINARY_OPERATORS]; + //R_RECEDENCE = (op is left-associative) ? PRECEDENCE + 1 : PRECEDENCE + + int precedenceCounter = 0; + + PRECEDENCE[AND] = precedenceCounter; + R_PRECEDENCE[AND] = PRECEDENCE[AND] + 1; + + PRECEDENCE[OR] = precedenceCounter; + R_PRECEDENCE[OR] = PRECEDENCE[OR] + 1; + + precedenceCounter++; + + PRECEDENCE[EQ] = precedenceCounter; + R_PRECEDENCE[EQ] = PRECEDENCE[EQ] + 1; + + PRECEDENCE[NEQ] = precedenceCounter; + R_PRECEDENCE[NEQ] = PRECEDENCE[NEQ] + 1; + + PRECEDENCE[LT] = precedenceCounter; + R_PRECEDENCE[LT] = PRECEDENCE[LT] + 1; + + PRECEDENCE[LEQ] = precedenceCounter; + R_PRECEDENCE[LEQ] = PRECEDENCE[LEQ] + 1; + + PRECEDENCE[GT] = precedenceCounter; + R_PRECEDENCE[GT] = PRECEDENCE[GT] + 1; + + PRECEDENCE[GEQ] = precedenceCounter; + R_PRECEDENCE[GEQ] = PRECEDENCE[GEQ] + 1; + + precedenceCounter++; + + PRECEDENCE[PLUS] = precedenceCounter; + R_PRECEDENCE[PLUS] = PRECEDENCE[PLUS] + 1; + + PRECEDENCE[MINUS] = precedenceCounter; + R_PRECEDENCE[MINUS] = PRECEDENCE[MINUS] + 1; + + precedenceCounter++; + + PRECEDENCE[MUL] = precedenceCounter; + R_PRECEDENCE[MUL] = PRECEDENCE[MUL] + 1; + + PRECEDENCE[DIV] = precedenceCounter; + R_PRECEDENCE[DIV] = PRECEDENCE[DIV] + 1; + + PRECEDENCE[MODULO] = precedenceCounter; + R_PRECEDENCE[MODULO] = PRECEDENCE[MODULO] + 1; + + precedenceCounter++; + + PRECEDENCE[CARROT] = precedenceCounter; + R_PRECEDENCE[CARROT] = PRECEDENCE[CARROT]; + + precedenceCounter++; + + UNARY_MINUS_PRECEDENCE = precedenceCounter; + UNARY_NOT_PRECEDENCE = precedenceCounter; + } + + private final int type; + private final String match; + + Token(int type) + { + this(type, ""); + } + + Token(int type, String match) + { + this.type = type; + this.match = match; + } + + int getType() + { + return type; + } + + String getMatch() + { + return match; + } + + @Override + public String toString() + { + return match; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + Token token = (Token) o; + + if (type != token.type) { + return false; + } + return !(match != null ? !match.equals(token.match) : token.match != null); + + } + + @Override + public int hashCode() + { + int result = type; + result = 31 * result + (match != null ? match.hashCode() : 0); + return result; + } +} diff --git a/common/src/test/java/io/druid/math/expr/EvalTest.java b/common/src/test/java/io/druid/math/expr/EvalTest.java new file mode 100644 index 00000000000..81976b5c138 --- /dev/null +++ b/common/src/test/java/io/druid/math/expr/EvalTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +/** + */ +public class EvalTest +{ + @Test + public void testDoubleEval() + { + Map bindings = new HashMap<>(); + bindings.put("x", 2.0d); + + Assert.assertEquals(2.0, Parser.parse("x").eval(bindings).doubleValue(), 0.0001); + + Assert.assertFalse(Parser.parse("1.0 and 0.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("1.0 and 2.0").eval(bindings).doubleValue() > 0.0); + + Assert.assertTrue(Parser.parse("1.0 or 0.0").eval(bindings).doubleValue() > 0.0); + Assert.assertFalse(Parser.parse("0.0 or 0.0").eval(bindings).doubleValue() > 0.0); + + Assert.assertTrue(Parser.parse("2.0 > 1.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("2.0 >= 2.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("1.0 < 2.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("2.0 <= 2.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("2.0 == 2.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("2.0 != 1.0").eval(bindings).doubleValue() > 0.0); + + Assert.assertEquals(3.5, Parser.parse("2.0 + 1.5").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(0.5, Parser.parse("2.0 - 1.5").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(3.0, Parser.parse("2.0 * 1.5").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(4.0, Parser.parse("2.0 / 0.5").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(0.2, Parser.parse("2.0 % 0.3").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(8.0, Parser.parse("2.0 ^ 3.0").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(-1.5, Parser.parse("-1.5").eval(bindings).doubleValue(), 0.0001); + + Assert.assertTrue(Parser.parse("!-1.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("!0.0").eval(bindings).doubleValue() > 0.0); + Assert.assertFalse(Parser.parse("!2.0").eval(bindings).doubleValue() > 0.0); + + Assert.assertEquals(2.0, Parser.parse("sqrt(4.0)").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(2.0, Parser.parse("if(1.0, 2.0, 3.0)").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(3.0, Parser.parse("if(0.0, 2.0, 3.0)").eval(bindings).doubleValue(), 0.0001); + } + + @Test + public void testLongEval() + { + Map bindings = new HashMap<>(); + bindings.put("x", 9223372036854775807l); + + Assert.assertEquals(9223372036854775807l, Parser.parse("x").eval(bindings).longValue()); + + Assert.assertFalse(Parser.parse("9223372036854775807 and 0").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775807 and 9223372036854775806").eval(bindings).longValue() > 0); + + Assert.assertTrue(Parser.parse("9223372036854775807 or 0").eval(bindings).longValue() > 0); + Assert.assertFalse(Parser.parse("-9223372036854775807 or -9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("-9223372036854775807 or 9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertFalse(Parser.parse("0 or 0").eval(bindings).longValue() > 0); + + Assert.assertTrue(Parser.parse("9223372036854775807 > 9223372036854775806").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775807 >= 9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775806 < 9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775807 <= 9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775807 == 9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775807 != 9223372036854775806").eval(bindings).longValue() > 0); + + Assert.assertEquals(9223372036854775807l, Parser.parse("9223372036854775806 + 1").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775806l, Parser.parse("9223372036854775807 - 1").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775806l, Parser.parse("4611686018427387903 * 2").eval(bindings).longValue()); + Assert.assertEquals(4611686018427387903l, Parser.parse("9223372036854775806 / 2").eval(bindings).longValue()); + Assert.assertEquals(7, Parser.parse("9223372036854775807 % 9223372036854775800").eval(bindings).longValue()); + Assert.assertEquals( 9223372030926249001l, Parser.parse("3037000499 ^ 2").eval(bindings).longValue()); + Assert.assertEquals(-9223372036854775807l, Parser.parse("-9223372036854775807").eval(bindings).longValue()); + + Assert.assertTrue(Parser.parse("!-9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("!0").eval(bindings).longValue() > 0); + Assert.assertFalse(Parser.parse("!9223372036854775807").eval(bindings).longValue() > 0); + + Assert.assertEquals(3037000499l, Parser.parse("sqrt(9223372036854775807)").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775807l, Parser.parse("if(9223372036854775807, 9223372036854775807, 9223372036854775806)").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775806l, Parser.parse("if(0, 9223372036854775807, 9223372036854775806)").eval(bindings).longValue()); + } +} diff --git a/common/src/test/java/io/druid/math/expr/LexerTest.java b/common/src/test/java/io/druid/math/expr/LexerTest.java new file mode 100644 index 00000000000..01d081348dd --- /dev/null +++ b/common/src/test/java/io/druid/math/expr/LexerTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +import org.junit.Assert; +import org.junit.Test; + +/** + */ +public class LexerTest +{ + @Test + public void testAllTokens() + { + testAllTokensAsserts("abcd<<=>>===!!=+-*/%^(),123 01.23"); + testAllTokensAsserts(" abcd < <= > >= == ! !=+ -*/ %^ () , 123 01.23 "); + } + + private void testAllTokensAsserts(String input) + { + Lexer lexer = new Lexer(input); + Assert.assertEquals(new Token(Token.IDENTIFIER, "abcd"), lexer.consume()); + Assert.assertEquals(new Token(Token.LT, "<"), lexer.consume()); + Assert.assertEquals(new Token(Token.LEQ, "<="), lexer.consume()); + Assert.assertEquals(new Token(Token.GT, ">"), lexer.consume()); + Assert.assertEquals(new Token(Token.GEQ, ">="), lexer.consume()); + Assert.assertEquals(new Token(Token.EQ, "=="), lexer.consume()); + Assert.assertEquals(new Token(Token.NOT, "!"), lexer.consume()); + Assert.assertEquals(new Token(Token.NEQ, "!="), lexer.consume()); + Assert.assertEquals(new Token(Token.PLUS, "+"), lexer.consume()); + Assert.assertEquals(new Token(Token.MINUS, "-"), lexer.consume()); + Assert.assertEquals(new Token(Token.MUL, "*"), lexer.consume()); + Assert.assertEquals(new Token(Token.DIV, "/"), lexer.consume()); + Assert.assertEquals(new Token(Token.MODULO, "%"), lexer.consume()); + Assert.assertEquals(new Token(Token.CARROT, "^"), lexer.consume()); + Assert.assertEquals(new Token(Token.LPAREN, "("), lexer.consume()); + Assert.assertEquals(new Token(Token.RPAREN, ")"), lexer.consume()); + Assert.assertEquals(new Token(Token.COMMA, ","), lexer.consume()); + Assert.assertEquals(new Token(Token.LONG, "123"), lexer.consume()); + Assert.assertEquals(new Token(Token.DOUBLE, "01.23"), lexer.consume()); + Assert.assertEquals(new Token(Token.EOF, ""), lexer.consume()); + Assert.assertEquals(new Token(Token.EOF, ""), lexer.consume()); + } +} diff --git a/common/src/test/java/io/druid/math/expr/ParserTest.java b/common/src/test/java/io/druid/math/expr/ParserTest.java new file mode 100644 index 00000000000..9d52fb1f0ee --- /dev/null +++ b/common/src/test/java/io/druid/math/expr/ParserTest.java @@ -0,0 +1,266 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +import org.junit.Assert; +import org.junit.Test; + +/** + */ +public class ParserTest +{ + @Test + public void testSimple() + { + String actual = Parser.parse("1").toString(); + String expected = "1"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleUnaryOps1() + { + String actual = Parser.parse("-x").toString(); + String expected = "-x"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("!x").toString(); + expected = "!x"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleUnaryOps2() + { + String actual = Parser.parse("-1").toString(); + String expected = "-1"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("--1").toString(); + expected = "--1"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("-1+2").toString(); + expected = "(+ -1 2)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("-1*2").toString(); + expected = "(* -1 2)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("-1^2").toString(); + expected = "(^ -1 2)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleLogicalOps1() + { + String actual = Parser.parse("x>y").toString(); + String expected = "(> x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x=y").toString(); + expected = "(>= x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x==y").toString(); + expected = "(== x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x!=y").toString(); + expected = "(!= x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x and y").toString(); + expected = "(and x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x or y").toString(); + expected = "(or x y)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleAdditivityOp1() + { + String actual = Parser.parse("x+y").toString(); + String expected = "(+ x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x-y").toString(); + expected = "(- x y)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleAdditivityOp2() + { + String actual = Parser.parse("x+y+z").toString(); + String expected = "(+ (+ x y) z)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x+y-z").toString(); + expected = "(- (+ x y) z)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x-y+z").toString(); + expected = "(+ (- x y) z)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x-y-z").toString(); + expected = "(- (- x y) z)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleMultiplicativeOp1() + { + String actual = Parser.parse("x*y").toString(); + String expected = "(* x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x/y").toString(); + expected = "(/ x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x%y").toString(); + expected = "(% x y)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleMultiplicativeOp2() + { + String actual = Parser.parse("1*2*3").toString(); + String expected = "(* (* 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1*2/3").toString(); + expected = "(/ (* 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1/2*3").toString(); + expected = "(* (/ 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1/2/3").toString(); + expected = "(/ (/ 1 2) 3)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleCarrot1() + { + String actual = Parser.parse("1^2").toString(); + String expected = "(^ 1 2)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleCarrot2() + { + String actual = Parser.parse("1^2^3").toString(); + String expected = "(^ 1 (^ 2 3))"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testMixed() + { + String actual = Parser.parse("1+2*3").toString(); + String expected = "(+ 1 (* 2 3))"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1+(2*3)").toString(); + Assert.assertEquals(expected, actual); + + actual = Parser.parse("(1+2)*3").toString(); + expected = "(* (+ 1 2) 3)"; + Assert.assertEquals(expected, actual); + + + actual = Parser.parse("1*2+3").toString(); + expected = "(+ (* 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("(1*2)+3").toString(); + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1*(2+3)").toString(); + expected = "(* 1 (+ 2 3))"; + Assert.assertEquals(expected, actual); + + + actual = Parser.parse("1+2^3)").toString(); + expected = "(+ 1 (^ 2 3))"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1+(2^3))").toString(); + expected = "(+ 1 (^ 2 3))"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("(1+2)^3)").toString(); + expected = "(^ (+ 1 2) 3)"; + Assert.assertEquals(expected, actual); + + + actual = Parser.parse("1^2+3)").toString(); + expected = "(+ (^ 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("(1^2)+3").toString(); + expected = "(+ (^ 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1^(2+3)").toString(); + expected = "(^ 1 (+ 2 3))"; + Assert.assertEquals(expected, actual); + + + actual = Parser.parse("1^2*3+4)").toString(); + expected = "(+ (* (^ 1 2) 3) 4)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("-1^-2*-3+-4)").toString(); + expected = "(+ (* (^ -1 -2) -3) -4)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testFunctions() + { + String actual = Parser.parse("sqrt(x)").toString(); + String expected = "(sqrt [x])"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("if(cond,then,else)").toString(); + expected = "(if [cond, then, else])"; + Assert.assertEquals(expected, actual); + } +} diff --git a/docs/content/misc/math-expr.md b/docs/content/misc/math-expr.md new file mode 100644 index 00000000000..c993e2a5276 --- /dev/null +++ b/docs/content/misc/math-expr.md @@ -0,0 +1,34 @@ +--- +layout: doc_page +--- + +This expression language supports following operators (listed in increasing order of precedence). + +|Operators|Description| +|---------|-----------| +|and,or|Binary Logical AND, OR| +|<, <=, >, >=, ==, !=|Binary Comparison| +|+, -|Binary additive| +|*, /, %|Binary multiplicative| +|^|Binary power op| +|!, -|Unary NOT and Minus| + +long and double data types are supported, number containing a dot is interpreted as a double or else a long. That means, always add a '.' to your number if you want it intepreted as a double value. +You can use variables inside the expression which must start with an alphabet or '_' or '$' and can contain a digit as well in subsequent characters. For logical operators, positive number is considered a boolean true and false otherwise. + +Also, following in-built functions are supported. + +|name|description| +|sqrt|sqrt(x) would return square root of x| +|if|if(predicate,then,else) would return `then` if predicate evaluates to a positive number or `else` is returned| + +### How to use? + +``` +Map bindings = new HashMap<>(); +bindings.put("x", 2); + +Number result = Parser.parse("x + 2").eval(bindings); +Assert.assertEquals(4, result.longValue()); +``` + From 308211cc1830c7f7257920145f337685e2ccc5a0 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Wed, 16 Dec 2015 01:03:10 -0600 Subject: [PATCH 2/3] math expression language with parser/lexer generated using ANTLR --- common/pom.xml | 15 + .../antlr4/io/druid/math/expr/antlr/Expr.g4 | 38 + .../main/java/io/druid/math/expr/Expr.java | 690 +++++++++++------- .../io/druid/math/expr/ExprListenerImpl.java | 312 ++++++++ .../main/java/io/druid/math/expr/Lexer.java | 229 ------ .../main/java/io/druid/math/expr/Parser.java | 117 +-- .../main/java/io/druid/math/expr/Token.java | 180 ----- .../java/io/druid/math/expr/EvalTest.java | 44 +- .../java/io/druid/math/expr/LexerTest.java | 61 -- .../java/io/druid/math/expr/ParserTest.java | 20 +- docs/content/misc/math-expr.md | 34 +- pom.xml | 6 +- 12 files changed, 843 insertions(+), 903 deletions(-) create mode 100644 common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4 create mode 100644 common/src/main/java/io/druid/math/expr/ExprListenerImpl.java delete mode 100644 common/src/main/java/io/druid/math/expr/Lexer.java delete mode 100644 common/src/main/java/io/druid/math/expr/Token.java delete mode 100644 common/src/test/java/io/druid/math/expr/LexerTest.java diff --git a/common/pom.xml b/common/pom.xml index 3d025e56f12..1df84723f8d 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -161,6 +161,10 @@ jets3t 0.9.4 + + org.antlr + antlr4-runtime + @@ -199,6 +203,17 @@ + + org.antlr + antlr4-maven-plugin + + + + antlr4 + + + + diff --git a/common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4 b/common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4 new file mode 100644 index 00000000000..f478911dd55 --- /dev/null +++ b/common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4 @@ -0,0 +1,38 @@ +grammar Expr; + +expr : ('-'|'!') expr # unaryOpExpr + | expr '^' expr # powOpExpr + | expr ('*'|'/'|'%') expr # mulDivModuloExpr + | expr ('+'|'-') expr # addSubExpr + | expr ('<'|'<='|'>'|'>='|'=='|'!=') expr # logicalOpExpr + | expr ('&&'|'||') expr # logicalAndOrExpr + | '(' expr ')' # nestedExpr + | IDENTIFIER '(' fnArgs? ')' # functionExpr + | IDENTIFIER # identifierExpr + | DOUBLE # doubleExpr + | LONG # longExpr + ; + +fnArgs : expr (',' expr)* # functionArgs + ; + +IDENTIFIER : [_$a-zA-Z][_$a-zA-Z0-9]* ; +LONG : [0-9]+ ; +DOUBLE : [0-9]+ '.' [0-9]* ; +WS : [ \t\r\n]+ -> skip ; + +MINUS : '-' ; +NOT : '!' ; +POW : '^' ; +MUL : '*' ; +DIV : '/' ; +MODULO : '%' ; +PLUS : '+' ; +LT : '<' ; +LEQ : '<=' ; +GT : '>' ; +GEQ : '>=' ; +EQ : '==' ; +NEQ : '!=' ; +AND : '&&' ; +OR : '||' ; diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java index 7e24c11d25f..2ee2cdae7c6 100644 --- a/common/src/main/java/io/druid/math/expr/Expr.java +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -31,209 +31,11 @@ public interface Expr Number eval(Map bindings); } -class SimpleExpr implements Expr -{ - private final Atom atom; - - public SimpleExpr(Atom atom) - { - this.atom = atom; - } - - @Override - public String toString() - { - return atom.toString(); - } - - @Override - public Number eval(Map bindings) - { - return atom.eval(bindings); - } -} - -class BinExpr implements Expr -{ - private final Token opToken; - private final Expr left; - private final Expr right; - - public BinExpr(Token opToken, Expr left, Expr right) - { - this.opToken = opToken; - this.left = left; - this.right = right; - } - - public Number eval(Map bindings) - { - switch(opToken.getType()) { - case Token.AND: - Number leftVal = left.eval(bindings); - Number rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { - long lval = leftVal.longValue(); - if (lval > 0) { - long rval = rightVal.longValue(); - return rval > 0 ? 1 : 0; - } else { - return 0; - } - } else { - double lval = leftVal.doubleValue(); - if (lval > 0) { - double rval = rightVal.doubleValue(); - return rval > 0 ? 1.0d : 0.0d; - } else { - return 0.0d; - } - } - case Token.OR: - leftVal = left.eval(bindings); - rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { - long lval = leftVal.longValue(); - if (lval > 0) { - return 1; - } else { - long rval = rightVal.longValue(); - return rval > 0 ? 1 : 0; - } - } else { - double lval = leftVal.doubleValue(); - if (lval > 0) { - return 1.0d; - } else { - double rval = rightVal.doubleValue(); - return rval > 0 ? 1.0d : 0.0d; - } - } - case Token.LT: - leftVal = left.eval(bindings); - rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { - return leftVal.longValue() < rightVal.longValue() ? 1 : 0; - } else { - return leftVal.doubleValue() < rightVal.doubleValue() ? 1.0d : 0.0d; - } - case Token.LEQ: - leftVal = left.eval(bindings); - rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { - return leftVal.longValue() <= rightVal.longValue() ? 1 : 0; - } else { - return leftVal.doubleValue() <= rightVal.doubleValue() ? 1.0d : 0.0d; - } - case Token.GT: - leftVal = left.eval(bindings); - rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { - return leftVal.longValue() > rightVal.longValue() ? 1 : 0; - } else { - return leftVal.doubleValue() > rightVal.doubleValue() ? 1.0d : 0.0d; - } - case Token.GEQ: - leftVal = left.eval(bindings); - rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { - return leftVal.longValue() >= rightVal.longValue() ? 1 : 0; - } else { - return leftVal.doubleValue() >= rightVal.doubleValue() ? 1.0d : 0.0d; - } - case Token.EQ: - leftVal = left.eval(bindings); - rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { - return leftVal.longValue() == rightVal.longValue() ? 1 : 0; - } else { - return leftVal.doubleValue() == rightVal.doubleValue() ? 1.0d : 0.0d; - } - case Token.NEQ: - leftVal = left.eval(bindings); - rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { - return leftVal.longValue() != rightVal.longValue() ? 1 : 0; - } else { - return leftVal.doubleValue() != rightVal.doubleValue() ? 1.0d : 0.0d; - } - case Token.PLUS: - leftVal = left.eval(bindings); - rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { - return leftVal.longValue() + rightVal.longValue(); - } else { - return leftVal.doubleValue() + rightVal.doubleValue(); - } - case Token.MINUS: - leftVal = left.eval(bindings); - rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { - return leftVal.longValue() - rightVal.longValue(); - } else { - return leftVal.doubleValue() - rightVal.doubleValue(); - } - case Token.MUL: - leftVal = left.eval(bindings); - rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { - return leftVal.longValue() * rightVal.longValue(); - } else { - return leftVal.doubleValue() * rightVal.doubleValue(); - } - case Token.DIV: - leftVal = left.eval(bindings); - rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { - return leftVal.longValue() / rightVal.longValue(); - } else { - return leftVal.doubleValue() / rightVal.doubleValue(); - } - case Token.MODULO: - leftVal = left.eval(bindings); - rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { - return leftVal.longValue() % rightVal.longValue(); - } else { - return leftVal.doubleValue() % rightVal.doubleValue(); - } - case Token.CARROT: - leftVal = left.eval(bindings); - rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { - return LongMath.pow(leftVal.longValue(), rightVal.intValue()); - } else { - return Math.pow(leftVal.doubleValue(), rightVal.doubleValue()); - } - default: - throw new RuntimeException("Unknown operator " + opToken.getMatch()); - } - } - - private boolean isLong(Number left, Number right) - { - return left instanceof Long && right instanceof Long; - } - - @Override - public String toString() - { - return "(" + opToken.getMatch() + " " + left + " " + right + ")"; - } -} - - - -interface Atom -{ - Number eval(Map bindings); -} - -class LongValueAtom implements Atom +class LongExpr implements Expr { private final long value; - public LongValueAtom(long value) + public LongExpr(long value) { this.value = value; } @@ -251,11 +53,11 @@ class LongValueAtom implements Atom } } -class DoubleValueAtom implements Atom +class DoubleExpr implements Expr { private final double value; - public DoubleValueAtom(double value) + public DoubleExpr(double value) { this.value = value; } @@ -273,11 +75,11 @@ class DoubleValueAtom implements Atom } } -class IdentifierAtom implements Atom +class IdentifierExpr implements Expr { - private final Token value; + private final String value; - public IdentifierAtom(Token value) + public IdentifierExpr(String value) { this.value = value; } @@ -285,100 +87,27 @@ class IdentifierAtom implements Atom @Override public String toString() { - return value.getMatch(); + return value; } @Override public Number eval(Map bindings) { - Number val = bindings.get(value.getMatch()); + Number val = bindings.get(value); if (val == null) { - throw new RuntimeException("No binding found for " + value.getMatch()); + throw new RuntimeException("No binding found for " + value); } else { - return val; + return val instanceof Long ? val : val.doubleValue(); } } } -class UnaryNotExprAtom implements Atom -{ - private final Expr expr; - - public UnaryNotExprAtom(Expr expr) - { - this.expr = expr; - } - - @Override - public String toString() - { - return "!" + expr.toString(); - } - - @Override - public Number eval(Map bindings) - { - Number valObj = expr.eval(bindings); - return valObj.doubleValue() > 0 ? 0.0d : 1.0d; - } -} - -class UnaryMinusExprAtom implements Atom -{ - private final Expr expr; - - public UnaryMinusExprAtom(Expr expr) - { - this.expr = expr; - } - - @Override - public String toString() - { - return "-" + expr.toString(); - } - - @Override - public Number eval(Map bindings) - { - Number valObj = expr.eval(bindings); - if (valObj instanceof Long) { - return -1 * valObj.longValue(); - } else { - return -1 * valObj.doubleValue(); - } - } -} - - -class NestedExprAtom implements Atom -{ - private final Expr expr; - - public NestedExprAtom(Expr expr) - { - this.expr = expr; - } - - @Override - public String toString() - { - return expr.toString(); - } - - @Override - public Number eval(Map bindings) - { - return expr.eval(bindings); - } -} - -class FunctionAtom implements Atom +class FunctionExpr implements Expr { private final String name; private final List args; - public FunctionAtom(String name, List args) + public FunctionExpr(String name, List args) { this.name = name; this.args = args; @@ -396,3 +125,396 @@ class FunctionAtom implements Atom return Parser.func.get(name).apply(args, bindings); } } + +class UnaryMinusExpr implements Expr +{ + private final Expr expr; + + UnaryMinusExpr(Expr expr) + { + this.expr = expr; + } + + @Override + public Number eval(Map bindings) + { + Number valObj = expr.eval(bindings); + if (valObj instanceof Long) { + return -1 * valObj.longValue(); + } else { + return -1 * valObj.doubleValue(); + } + } + + @Override + public String toString() + { + return "-" + expr.toString(); + } +} + +class UnaryNotExpr implements Expr +{ + private final Expr expr; + + UnaryNotExpr(Expr expr) + { + this.expr = expr; + } + + @Override + public Number eval(Map bindings) + { + Number valObj = expr.eval(bindings); + return valObj.doubleValue() > 0 ? 0.0d : 1.0d; + } + + @Override + public String toString() + { + return "!" + expr.toString(); + } +} + +abstract class BinaryOpExprBase implements Expr +{ + protected final String op; + protected final Expr left; + protected final Expr right; + + public BinaryOpExprBase(String op, Expr left, Expr right) + { + this.op = op; + this.left = left; + this.right = right; + } + + protected boolean isLong(Number left, Number right) + { + return left instanceof Long && right instanceof Long; + } + + @Override + public String toString() + { + return "(" + op + " " + left + " " + right + ")"; + } +} + +class BinMinusExpr extends BinaryOpExprBase +{ + + BinMinusExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() - rightVal.longValue(); + } else { + return leftVal.doubleValue() - rightVal.doubleValue(); + } + } +} + +class BinPowExpr extends BinaryOpExprBase +{ + + BinPowExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return LongMath.pow(leftVal.longValue(), rightVal.intValue()); + } else { + return Math.pow(leftVal.doubleValue(), rightVal.doubleValue()); + } + } +} + +class BinMulExpr extends BinaryOpExprBase +{ + + BinMulExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() * rightVal.longValue(); + } else { + return leftVal.doubleValue() * rightVal.doubleValue(); + } + } +} + +class BinDivExpr extends BinaryOpExprBase +{ + + BinDivExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() / rightVal.longValue(); + } else { + return leftVal.doubleValue() / rightVal.doubleValue(); + } + } +} + +class BinModuloExpr extends BinaryOpExprBase +{ + + BinModuloExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() % rightVal.longValue(); + } else { + return leftVal.doubleValue() % rightVal.doubleValue(); + } + } +} + +class BinPlusExpr extends BinaryOpExprBase +{ + + BinPlusExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() + rightVal.longValue(); + } else { + return leftVal.doubleValue() + rightVal.doubleValue(); + } + } +} + +class BinLtExpr extends BinaryOpExprBase +{ + + BinLtExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() < rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() < rightVal.doubleValue() ? 1.0d : 0.0d; + } + } +} + +class BinLeqExpr extends BinaryOpExprBase +{ + + BinLeqExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() <= rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() <= rightVal.doubleValue() ? 1.0d : 0.0d; + } + } +} + +class BinGtExpr extends BinaryOpExprBase +{ + + BinGtExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() > rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() > rightVal.doubleValue() ? 1.0d : 0.0d; + } + } +} + +class BinGeqExpr extends BinaryOpExprBase +{ + + BinGeqExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() >= rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() >= rightVal.doubleValue() ? 1.0d : 0.0d; + } + } +} + +class BinEqExpr extends BinaryOpExprBase +{ + + BinEqExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() == rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() == rightVal.doubleValue() ? 1.0d : 0.0d; + } + } +} + +class BinNeqExpr extends BinaryOpExprBase +{ + + BinNeqExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() != rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() != rightVal.doubleValue() ? 1.0d : 0.0d; + } + } +} + +class BinAndExpr extends BinaryOpExprBase +{ + + BinAndExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + long lval = leftVal.longValue(); + if (lval > 0) { + long rval = rightVal.longValue(); + return rval > 0 ? 1 : 0; + } else { + return 0; + } + } else { + double lval = leftVal.doubleValue(); + if (lval > 0) { + double rval = rightVal.doubleValue(); + return rval > 0 ? 1.0d : 0.0d; + } else { + return 0.0d; + } + } + } +} + +class BinOrExpr extends BinaryOpExprBase +{ + + BinOrExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + long lval = leftVal.longValue(); + if (lval > 0) { + return 1; + } else { + long rval = rightVal.longValue(); + return rval > 0 ? 1 : 0; + } + } else { + double lval = leftVal.doubleValue(); + if (lval > 0) { + return 1.0d; + } else { + double rval = rightVal.doubleValue(); + return rval > 0 ? 1.0d : 0.0d; + } + } + } +} diff --git a/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java b/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java new file mode 100644 index 00000000000..39fecc779c9 --- /dev/null +++ b/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java @@ -0,0 +1,312 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +import io.druid.math.expr.antlr.ExprBaseListener; +import io.druid.math.expr.antlr.ExprParser; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.TerminalNode; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + */ +public class ExprListenerImpl extends ExprBaseListener +{ + private final Map nodes; + private final ParseTree rootNodeKey; + + ExprListenerImpl(ParseTree rootNodeKey) + { + this.rootNodeKey = rootNodeKey; + this.nodes = new HashMap<>(); + } + + Expr getAST() + { + return (Expr) nodes.get(rootNodeKey); + } + + @Override + public void exitUnaryOpExpr(ExprParser.UnaryOpExprContext ctx) + { + int opCode = ((TerminalNode) ctx.getChild(0)).getSymbol().getType(); + switch (opCode) { + case ExprParser.MINUS: + nodes.put(ctx, new UnaryMinusExpr((Expr) nodes.get(ctx.getChild(1)))); + break; + case ExprParser.NOT: + nodes.put(ctx, new UnaryNotExpr((Expr) nodes.get(ctx.getChild(1)))); + break; + default: + throw new RuntimeException("Unrecognized unary operator " + ctx.getChild(0).getText()); + } + } + + @Override + public void exitDoubleExpr(ExprParser.DoubleExprContext ctx) + { + nodes.put( + ctx, + new DoubleExpr(Double.parseDouble(ctx.getText())) + ); + } + + @Override + public void exitAddSubExpr(ExprParser.AddSubExprContext ctx) + { + int opCode = ((TerminalNode) ctx.getChild(1)).getSymbol().getType(); + switch (opCode) { + case ExprParser.PLUS: + nodes.put( + ctx, + new BinPlusExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.MINUS: + nodes.put( + ctx, + new BinMinusExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + default: + throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText()); + } + } + + @Override + public void exitLongExpr(ExprParser.LongExprContext ctx) + { + nodes.put( + ctx, + new LongExpr(Long.parseLong(ctx.getText())) + ); + } + + @Override + public void exitLogicalAndOrExpr(ExprParser.LogicalAndOrExprContext ctx) + { + int opCode = ((TerminalNode) ctx.getChild(1)).getSymbol().getType(); + switch (opCode) { + case ExprParser.AND: + nodes.put( + ctx, + new BinAndExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.OR: + nodes.put( + ctx, + new BinOrExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + default: + throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText()); + } + } + + @Override + public void exitNestedExpr(ExprParser.NestedExprContext ctx) + { + nodes.put(ctx, nodes.get(ctx.getChild(1))); + } + + @Override + public void exitLogicalOpExpr(ExprParser.LogicalOpExprContext ctx) + { + int opCode = ((TerminalNode) ctx.getChild(1)).getSymbol().getType(); + switch (opCode) { + case ExprParser.LT: + nodes.put( + ctx, + new BinLtExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.LEQ: + nodes.put( + ctx, + new BinLeqExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.GT: + nodes.put( + ctx, + new BinGtExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.GEQ: + nodes.put( + ctx, + new BinGeqExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.EQ: + nodes.put( + ctx, + new BinEqExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.NEQ: + nodes.put( + ctx, + new BinNeqExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + default: + throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText()); + } + } + + @Override + public void exitMulDivModuloExpr(ExprParser.MulDivModuloExprContext ctx) + { + int opCode = ((TerminalNode) ctx.getChild(1)).getSymbol().getType(); + switch (opCode) { + case ExprParser.MUL: + nodes.put( + ctx, + new BinMulExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.DIV: + nodes.put( + ctx, + new BinDivExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.MODULO: + nodes.put( + ctx, + new BinModuloExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + default: + throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText()); + } + } + + @Override + public void exitPowOpExpr(ExprParser.PowOpExprContext ctx) + { + nodes.put( + ctx, + new BinPowExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + } + + @Override + public void exitFunctionExpr(ExprParser.FunctionExprContext ctx) + { + String fnName = ctx.getChild(0).getText(); + if (!Parser.func.containsKey(fnName)) { + throw new RuntimeException("function " + fnName + " is not defined."); + } + + List args = ctx.getChildCount() > 3 ? (List) nodes.get(ctx.getChild(2)) : Collections.emptyList(); + nodes.put( + ctx, + new FunctionExpr(fnName, args) + ); + } + + @Override + public void exitIdentifierExpr(ExprParser.IdentifierExprContext ctx) + { + nodes.put( + ctx, + new IdentifierExpr(ctx.getText()) + ); + } + + @Override + public void exitFunctionArgs(ExprParser.FunctionArgsContext ctx) + { + List args = new ArrayList<>(); + args.add((Expr) nodes.get(ctx.getChild(0))); + + if (ctx.getChildCount() > 1) { + for (int i = 1; i <= ctx.getChildCount() / 2; i++) { + args.add((Expr) nodes.get(ctx.getChild(2 * i))); + } + } + + nodes.put(ctx, args); + } +} diff --git a/common/src/main/java/io/druid/math/expr/Lexer.java b/common/src/main/java/io/druid/math/expr/Lexer.java deleted file mode 100644 index fc55a689ea6..00000000000 --- a/common/src/main/java/io/druid/math/expr/Lexer.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.math.expr; - -class Lexer -{ - private String input; - private int currPos; - - private Token next; - - Lexer(String input) - { - this.input = input; - currPos = 0; - - next = nextToken(); - } - - Token peek() - { - return next; - } - - Token consume() - { - Token old = next; - next = nextToken(); - return old; - } - - private Token nextToken() - { - if (currPos >= input.length()) { - return new Token(Token.EOF); - } - - char c = input.charAt(currPos); - while(c == ' ') { - currPos++; - if (currPos >= input.length()) { - return new Token(Token.EOF); - } - c = input.charAt(currPos); - } - - switch(c) - { - case '<': - currPos++; - if (currPos < input.length() && input.charAt(currPos) == '=') { - currPos++; - return new Token(Token.LEQ, "<="); - } else { - return new Token(Token.LT, "<"); - } - case '>': - currPos++; - if (currPos < input.length() && input.charAt(currPos) == '=') { - currPos++; - return new Token(Token.GEQ, ">="); - } else { - return new Token(Token.GT, ">"); - } - case '=': - currPos++; - if (currPos < input.length() && input.charAt(currPos) == '=') { - currPos++; - return new Token(Token.EQ, "=="); - } else { - throw new IllegalArgumentException("unknown operator '='"); - } - case '!': - currPos++; - if (currPos < input.length() && input.charAt(currPos) == '=') { - currPos++; - return new Token(Token.NEQ, "!="); - } else { - return new Token(Token.NOT, "!"); - } - - case '+': - currPos++; - return new Token(Token.PLUS, "+"); - case '-': - currPos++; - return new Token(Token.MINUS, "-"); - - case '*': - currPos++; - return new Token(Token.MUL, "*"); - case '/': - currPos++; - return new Token(Token.DIV, "/"); - case '%': - currPos++; - return new Token(Token.MODULO, "%"); - - case '^': - currPos++; - return new Token(Token.CARROT, "^"); - - case '(': - currPos++; - return new Token(Token.LPAREN, "("); - case ')': - currPos++; - return new Token(Token.RPAREN, ")"); - case ',': - currPos++; - return new Token(Token.COMMA, ","); - - default: - if (isNumberStartingChar(c)) { - return parseNumber(); - } else if (isIdentifierStartingChar(c)){ - return parseIdentifierOrKeyword(); - } else { - throw new RuntimeException("Illegal expression " + toString()); - } - } - } - - private boolean isNumberStartingChar(char c) - { - return c >= '0' && c <= '9'; - } - - private boolean isIdentifierStartingChar(char c) - { - return (c >= 'a' && c <= 'z') || - (c >= 'A' && c <= 'Z') || - c == '$' || - c == '_'; - } - - private boolean isIdentifierChar(char c) - { - return (c >= 'a' && c <= 'z') || - (c >= 'A' && c <= 'Z') || - (c >= '0' && c <= '9') || - c == '$' || - c == '_'; - } - - private Token parseIdentifierOrKeyword() - { - StringBuilder sb = new StringBuilder(); - char c = input.charAt(currPos++); - while (isIdentifierChar(c)) { - sb.append(c); - - if (currPos < input.length()) { - c = input.charAt(currPos++); - } else { - currPos++; - break; - } - }; - - currPos--; - - String str = sb.toString(); - if(str.equals("and")) { - return new Token(Token.AND, str); - } else if(str.equals("or")) { - return new Token(Token.OR, str); - } else { - return new Token(Token.IDENTIFIER, sb.toString()); - } - } - - // Numbers - // long : [0-9]+ - // double : [0-9]+.[0-9]+ - private Token parseNumber() - { - boolean isLong = true; - StringBuilder sb = new StringBuilder(); - char c = input.charAt(currPos++); - - while ( - ('0' <= c && c <= '9') || c == '.' - ) { - if (c == '.') { - isLong = false; - } - - sb.append(c); - - if (currPos < input.length()) { - c = input.charAt(currPos++); - } else { - currPos++; - break; - } - }; - - currPos--; - if (isLong) { - return new Token(Token.LONG, sb.toString()); - } else { - return new Token(Token.DOUBLE, sb.toString()); - } - } - - @Override - public String toString() - { - return "at " + currPos + ", " + input; - } -} diff --git a/common/src/main/java/io/druid/math/expr/Parser.java b/common/src/main/java/io/druid/math/expr/Parser.java index 588db098f45..9c962e11668 100644 --- a/common/src/main/java/io/druid/math/expr/Parser.java +++ b/common/src/main/java/io/druid/math/expr/Parser.java @@ -20,108 +20,37 @@ package io.druid.math.expr; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; +import com.google.common.collect.ImmutableMap; +import io.druid.math.expr.antlr.ExprLexer; +import io.druid.math.expr.antlr.ExprParser; +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.ParseTreeWalker; + import java.util.Map; public class Parser { - static final Map func = new HashMap<>(); + static final Map func; static { - func.put("sqrt", new SqrtFunc()); - func.put("if", new ConditionFunc()); + func = ImmutableMap.builder() + .put("sqrt", new SqrtFunc()) + .put("if", new ConditionFunc()) + .build(); } - private final Lexer lexer; - - private Parser(Lexer lexer) + public static Expr parse(String in) { - this.lexer = lexer; - } - - public static Expr parse(String input) - { - return new Parser(new Lexer(input)).parseExpr(0); - } - - private Expr parseExpr(int p) - { - Expr result = new SimpleExpr(parseAtom()); - - Token t = lexer.peek(); - while ( - t.getType() < Token.NUM_BINARY_OPERATORS - && - Token.PRECEDENCE[t.getType()] >= p - ) { - lexer.consume(); - result = new BinExpr( - t, - result, - parseExpr(Token.R_PRECEDENCE[t.getType()]) - ); - - t = lexer.peek(); - } - - return result; - } - - private Atom parseAtom() - { - Token t = lexer.peek(); - - switch(t.getType()) { - case Token.IDENTIFIER: - lexer.consume(); - String id = t.getMatch(); - - if (func.containsKey(id)) { - expect(Token.LPAREN); - List args = new ArrayList<>(); - t = lexer.peek(); - while(t.getType() != Token.RPAREN) { - args.add(parseExpr(0)); - t = lexer.peek(); - if (t.getType() == Token.COMMA) { - lexer.consume(); - } - } - expect(Token.RPAREN); - return new FunctionAtom(id, args); - } - - return new IdentifierAtom(t); - case Token.LONG: - lexer.consume(); - return new LongValueAtom(Long.valueOf(t.getMatch())); - case Token.DOUBLE: - lexer.consume(); - return new DoubleValueAtom(Double.valueOf(t.getMatch())); - case Token.MINUS: - lexer.consume(); - return new UnaryMinusExprAtom(parseExpr(Token.UNARY_MINUS_PRECEDENCE)); - case Token.NOT: - lexer.consume(); - return new UnaryNotExprAtom(parseExpr(Token.UNARY_NOT_PRECEDENCE)); - case Token.LPAREN: - lexer.consume(); - Expr expression = parseExpr(0); - if(lexer.consume().getType() == Token.RPAREN) { - return new NestedExprAtom(expression); - } - default: - throw new RuntimeException("Invalid token found " + t + " in input " + lexer); - } - } - - private void expect(int type) - { - Token t = lexer.consume(); - if(t.getType() != type) { - throw new RuntimeException("Invalid token found " + t + " in input " + lexer); - } + ExprLexer lexer = new ExprLexer(new ANTLRInputStream(in)); + CommonTokenStream tokens = new CommonTokenStream(lexer); + ExprParser parser = new ExprParser(tokens); + parser.setBuildParseTree(true); + ParseTree parseTree = parser.expr(); + ParseTreeWalker walker = new ParseTreeWalker(); + ExprListenerImpl listener = new ExprListenerImpl(parseTree); + walker.walk(listener, parseTree); + return listener.getAST(); } } diff --git a/common/src/main/java/io/druid/math/expr/Token.java b/common/src/main/java/io/druid/math/expr/Token.java deleted file mode 100644 index 25cf6ac3e34..00000000000 --- a/common/src/main/java/io/druid/math/expr/Token.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.math.expr; - -/** - */ -class Token -{ - static final int AND = 0; - static final int OR = 1; - - static final int LT = 2; - static final int LEQ = 3; - static final int GT = 4; - static final int GEQ = 5; - static final int EQ = 6; - static final int NEQ = 7; - - static final int PLUS = 8; - static final int MINUS = 9; - - static final int MUL = 10; - static final int DIV = 11; - static final int MODULO = 12; - - static final int CARROT = 13; - static final int NUM_BINARY_OPERATORS = 14; - - static final int LPAREN = 51; - static final int RPAREN = 52; - static final int COMMA = 53; - static final int NOT = 54; - static final int IDENTIFIER = 55; - static final int LONG = 56; - static final int DOUBLE = 57; - - static final int EOF = 100; - - static final int[] PRECEDENCE; - static final int[] R_PRECEDENCE; - static final int UNARY_MINUS_PRECEDENCE; - static final int UNARY_NOT_PRECEDENCE; - - static { - PRECEDENCE = new int[NUM_BINARY_OPERATORS]; - R_PRECEDENCE = new int[NUM_BINARY_OPERATORS]; - //R_RECEDENCE = (op is left-associative) ? PRECEDENCE + 1 : PRECEDENCE - - int precedenceCounter = 0; - - PRECEDENCE[AND] = precedenceCounter; - R_PRECEDENCE[AND] = PRECEDENCE[AND] + 1; - - PRECEDENCE[OR] = precedenceCounter; - R_PRECEDENCE[OR] = PRECEDENCE[OR] + 1; - - precedenceCounter++; - - PRECEDENCE[EQ] = precedenceCounter; - R_PRECEDENCE[EQ] = PRECEDENCE[EQ] + 1; - - PRECEDENCE[NEQ] = precedenceCounter; - R_PRECEDENCE[NEQ] = PRECEDENCE[NEQ] + 1; - - PRECEDENCE[LT] = precedenceCounter; - R_PRECEDENCE[LT] = PRECEDENCE[LT] + 1; - - PRECEDENCE[LEQ] = precedenceCounter; - R_PRECEDENCE[LEQ] = PRECEDENCE[LEQ] + 1; - - PRECEDENCE[GT] = precedenceCounter; - R_PRECEDENCE[GT] = PRECEDENCE[GT] + 1; - - PRECEDENCE[GEQ] = precedenceCounter; - R_PRECEDENCE[GEQ] = PRECEDENCE[GEQ] + 1; - - precedenceCounter++; - - PRECEDENCE[PLUS] = precedenceCounter; - R_PRECEDENCE[PLUS] = PRECEDENCE[PLUS] + 1; - - PRECEDENCE[MINUS] = precedenceCounter; - R_PRECEDENCE[MINUS] = PRECEDENCE[MINUS] + 1; - - precedenceCounter++; - - PRECEDENCE[MUL] = precedenceCounter; - R_PRECEDENCE[MUL] = PRECEDENCE[MUL] + 1; - - PRECEDENCE[DIV] = precedenceCounter; - R_PRECEDENCE[DIV] = PRECEDENCE[DIV] + 1; - - PRECEDENCE[MODULO] = precedenceCounter; - R_PRECEDENCE[MODULO] = PRECEDENCE[MODULO] + 1; - - precedenceCounter++; - - PRECEDENCE[CARROT] = precedenceCounter; - R_PRECEDENCE[CARROT] = PRECEDENCE[CARROT]; - - precedenceCounter++; - - UNARY_MINUS_PRECEDENCE = precedenceCounter; - UNARY_NOT_PRECEDENCE = precedenceCounter; - } - - private final int type; - private final String match; - - Token(int type) - { - this(type, ""); - } - - Token(int type, String match) - { - this.type = type; - this.match = match; - } - - int getType() - { - return type; - } - - String getMatch() - { - return match; - } - - @Override - public String toString() - { - return match; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - Token token = (Token) o; - - if (type != token.type) { - return false; - } - return !(match != null ? !match.equals(token.match) : token.match != null); - - } - - @Override - public int hashCode() - { - int result = type; - result = 31 * result + (match != null ? match.hashCode() : 0); - return result; - } -} diff --git a/common/src/test/java/io/druid/math/expr/EvalTest.java b/common/src/test/java/io/druid/math/expr/EvalTest.java index 81976b5c138..bb24ec826e3 100644 --- a/common/src/test/java/io/druid/math/expr/EvalTest.java +++ b/common/src/test/java/io/druid/math/expr/EvalTest.java @@ -37,11 +37,11 @@ public class EvalTest Assert.assertEquals(2.0, Parser.parse("x").eval(bindings).doubleValue(), 0.0001); - Assert.assertFalse(Parser.parse("1.0 and 0.0").eval(bindings).doubleValue() > 0.0); - Assert.assertTrue(Parser.parse("1.0 and 2.0").eval(bindings).doubleValue() > 0.0); + Assert.assertFalse(Parser.parse("1.0 && 0.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("1.0 && 2.0").eval(bindings).doubleValue() > 0.0); - Assert.assertTrue(Parser.parse("1.0 or 0.0").eval(bindings).doubleValue() > 0.0); - Assert.assertFalse(Parser.parse("0.0 or 0.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("1.0 || 0.0").eval(bindings).doubleValue() > 0.0); + Assert.assertFalse(Parser.parse("0.0 || 0.0").eval(bindings).doubleValue() > 0.0); Assert.assertTrue(Parser.parse("2.0 > 1.0").eval(bindings).doubleValue() > 0.0); Assert.assertTrue(Parser.parse("2.0 >= 2.0").eval(bindings).doubleValue() > 0.0); @@ -71,17 +71,17 @@ public class EvalTest public void testLongEval() { Map bindings = new HashMap<>(); - bindings.put("x", 9223372036854775807l); + bindings.put("x", 9223372036854775807L); - Assert.assertEquals(9223372036854775807l, Parser.parse("x").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775807L, Parser.parse("x").eval(bindings).longValue()); - Assert.assertFalse(Parser.parse("9223372036854775807 and 0").eval(bindings).longValue() > 0); - Assert.assertTrue(Parser.parse("9223372036854775807 and 9223372036854775806").eval(bindings).longValue() > 0); + Assert.assertFalse(Parser.parse("9223372036854775807 && 0").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775807 && 9223372036854775806").eval(bindings).longValue() > 0); - Assert.assertTrue(Parser.parse("9223372036854775807 or 0").eval(bindings).longValue() > 0); - Assert.assertFalse(Parser.parse("-9223372036854775807 or -9223372036854775807").eval(bindings).longValue() > 0); - Assert.assertTrue(Parser.parse("-9223372036854775807 or 9223372036854775807").eval(bindings).longValue() > 0); - Assert.assertFalse(Parser.parse("0 or 0").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775807 || 0").eval(bindings).longValue() > 0); + Assert.assertFalse(Parser.parse("-9223372036854775807 || -9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("-9223372036854775807 || 9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertFalse(Parser.parse("0 || 0").eval(bindings).longValue() > 0); Assert.assertTrue(Parser.parse("9223372036854775807 > 9223372036854775806").eval(bindings).longValue() > 0); Assert.assertTrue(Parser.parse("9223372036854775807 >= 9223372036854775807").eval(bindings).longValue() > 0); @@ -90,20 +90,20 @@ public class EvalTest Assert.assertTrue(Parser.parse("9223372036854775807 == 9223372036854775807").eval(bindings).longValue() > 0); Assert.assertTrue(Parser.parse("9223372036854775807 != 9223372036854775806").eval(bindings).longValue() > 0); - Assert.assertEquals(9223372036854775807l, Parser.parse("9223372036854775806 + 1").eval(bindings).longValue()); - Assert.assertEquals(9223372036854775806l, Parser.parse("9223372036854775807 - 1").eval(bindings).longValue()); - Assert.assertEquals(9223372036854775806l, Parser.parse("4611686018427387903 * 2").eval(bindings).longValue()); - Assert.assertEquals(4611686018427387903l, Parser.parse("9223372036854775806 / 2").eval(bindings).longValue()); - Assert.assertEquals(7, Parser.parse("9223372036854775807 % 9223372036854775800").eval(bindings).longValue()); - Assert.assertEquals( 9223372030926249001l, Parser.parse("3037000499 ^ 2").eval(bindings).longValue()); - Assert.assertEquals(-9223372036854775807l, Parser.parse("-9223372036854775807").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775807L, Parser.parse("9223372036854775806 + 1").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775806L, Parser.parse("9223372036854775807 - 1").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775806L, Parser.parse("4611686018427387903 * 2").eval(bindings).longValue()); + Assert.assertEquals(4611686018427387903L, Parser.parse("9223372036854775806 / 2").eval(bindings).longValue()); + Assert.assertEquals(7L, Parser.parse("9223372036854775807 % 9223372036854775800").eval(bindings).longValue()); + Assert.assertEquals( 9223372030926249001L, Parser.parse("3037000499 ^ 2").eval(bindings).longValue()); + Assert.assertEquals(-9223372036854775807L, Parser.parse("-9223372036854775807").eval(bindings).longValue()); Assert.assertTrue(Parser.parse("!-9223372036854775807").eval(bindings).longValue() > 0); Assert.assertTrue(Parser.parse("!0").eval(bindings).longValue() > 0); Assert.assertFalse(Parser.parse("!9223372036854775807").eval(bindings).longValue() > 0); - Assert.assertEquals(3037000499l, Parser.parse("sqrt(9223372036854775807)").eval(bindings).longValue()); - Assert.assertEquals(9223372036854775807l, Parser.parse("if(9223372036854775807, 9223372036854775807, 9223372036854775806)").eval(bindings).longValue()); - Assert.assertEquals(9223372036854775806l, Parser.parse("if(0, 9223372036854775807, 9223372036854775806)").eval(bindings).longValue()); + Assert.assertEquals(3037000499L, Parser.parse("sqrt(9223372036854775807)").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775807L, Parser.parse("if(9223372036854775807, 9223372036854775807, 9223372036854775806)").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775806L, Parser.parse("if(0, 9223372036854775807, 9223372036854775806)").eval(bindings).longValue()); } } diff --git a/common/src/test/java/io/druid/math/expr/LexerTest.java b/common/src/test/java/io/druid/math/expr/LexerTest.java deleted file mode 100644 index 01d081348dd..00000000000 --- a/common/src/test/java/io/druid/math/expr/LexerTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.math.expr; - -import org.junit.Assert; -import org.junit.Test; - -/** - */ -public class LexerTest -{ - @Test - public void testAllTokens() - { - testAllTokensAsserts("abcd<<=>>===!!=+-*/%^(),123 01.23"); - testAllTokensAsserts(" abcd < <= > >= == ! !=+ -*/ %^ () , 123 01.23 "); - } - - private void testAllTokensAsserts(String input) - { - Lexer lexer = new Lexer(input); - Assert.assertEquals(new Token(Token.IDENTIFIER, "abcd"), lexer.consume()); - Assert.assertEquals(new Token(Token.LT, "<"), lexer.consume()); - Assert.assertEquals(new Token(Token.LEQ, "<="), lexer.consume()); - Assert.assertEquals(new Token(Token.GT, ">"), lexer.consume()); - Assert.assertEquals(new Token(Token.GEQ, ">="), lexer.consume()); - Assert.assertEquals(new Token(Token.EQ, "=="), lexer.consume()); - Assert.assertEquals(new Token(Token.NOT, "!"), lexer.consume()); - Assert.assertEquals(new Token(Token.NEQ, "!="), lexer.consume()); - Assert.assertEquals(new Token(Token.PLUS, "+"), lexer.consume()); - Assert.assertEquals(new Token(Token.MINUS, "-"), lexer.consume()); - Assert.assertEquals(new Token(Token.MUL, "*"), lexer.consume()); - Assert.assertEquals(new Token(Token.DIV, "/"), lexer.consume()); - Assert.assertEquals(new Token(Token.MODULO, "%"), lexer.consume()); - Assert.assertEquals(new Token(Token.CARROT, "^"), lexer.consume()); - Assert.assertEquals(new Token(Token.LPAREN, "("), lexer.consume()); - Assert.assertEquals(new Token(Token.RPAREN, ")"), lexer.consume()); - Assert.assertEquals(new Token(Token.COMMA, ","), lexer.consume()); - Assert.assertEquals(new Token(Token.LONG, "123"), lexer.consume()); - Assert.assertEquals(new Token(Token.DOUBLE, "01.23"), lexer.consume()); - Assert.assertEquals(new Token(Token.EOF, ""), lexer.consume()); - Assert.assertEquals(new Token(Token.EOF, ""), lexer.consume()); - } -} diff --git a/common/src/test/java/io/druid/math/expr/ParserTest.java b/common/src/test/java/io/druid/math/expr/ParserTest.java index 9d52fb1f0ee..512bcbd0426 100644 --- a/common/src/test/java/io/druid/math/expr/ParserTest.java +++ b/common/src/test/java/io/druid/math/expr/ParserTest.java @@ -97,12 +97,12 @@ public class ParserTest expected = "(!= x y)"; Assert.assertEquals(expected, actual); - actual = Parser.parse("x and y").toString(); - expected = "(and x y)"; + actual = Parser.parse("x && y").toString(); + expected = "(&& x y)"; Assert.assertEquals(expected, actual); - actual = Parser.parse("x or y").toString(); - expected = "(or x y)"; + actual = Parser.parse("x || y").toString(); + expected = "(|| x y)"; Assert.assertEquals(expected, actual); } @@ -217,20 +217,20 @@ public class ParserTest Assert.assertEquals(expected, actual); - actual = Parser.parse("1+2^3)").toString(); + actual = Parser.parse("1+2^3").toString(); expected = "(+ 1 (^ 2 3))"; Assert.assertEquals(expected, actual); - actual = Parser.parse("1+(2^3))").toString(); + actual = Parser.parse("1+(2^3)").toString(); expected = "(+ 1 (^ 2 3))"; Assert.assertEquals(expected, actual); - actual = Parser.parse("(1+2)^3)").toString(); + actual = Parser.parse("(1+2)^3").toString(); expected = "(^ (+ 1 2) 3)"; Assert.assertEquals(expected, actual); - actual = Parser.parse("1^2+3)").toString(); + actual = Parser.parse("1^2+3").toString(); expected = "(+ (^ 1 2) 3)"; Assert.assertEquals(expected, actual); @@ -243,11 +243,11 @@ public class ParserTest Assert.assertEquals(expected, actual); - actual = Parser.parse("1^2*3+4)").toString(); + actual = Parser.parse("1^2*3+4").toString(); expected = "(+ (* (^ 1 2) 3) 4)"; Assert.assertEquals(expected, actual); - actual = Parser.parse("-1^-2*-3+-4)").toString(); + actual = Parser.parse("-1^-2*-3+-4").toString(); expected = "(+ (* (^ -1 -2) -3) -4)"; Assert.assertEquals(expected, actual); } diff --git a/docs/content/misc/math-expr.md b/docs/content/misc/math-expr.md index c993e2a5276..b338bf37c21 100644 --- a/docs/content/misc/math-expr.md +++ b/docs/content/misc/math-expr.md @@ -2,33 +2,27 @@ layout: doc_page --- -This expression language supports following operators (listed in increasing order of precedence). +This expression language supports the following operators (listed in decreasing order of precedence). |Operators|Description| |---------|-----------| -|and,or|Binary Logical AND, OR| -|<, <=, >, >=, ==, !=|Binary Comparison| -|+, -|Binary additive| -|*, /, %|Binary multiplicative| -|^|Binary power op| |!, -|Unary NOT and Minus| +|^|Binary power op| +|*, /, %|Binary multiplicative| +|+, -|Binary additive| +|<, <=, >, >=, ==, !=|Binary Comparison| +|&&,\|\||Binary Logical AND, OR| -long and double data types are supported, number containing a dot is interpreted as a double or else a long. That means, always add a '.' to your number if you want it intepreted as a double value. -You can use variables inside the expression which must start with an alphabet or '_' or '$' and can contain a digit as well in subsequent characters. For logical operators, positive number is considered a boolean true and false otherwise. +Long and double data types are supported. If a number contains a dot, it is interpreted as a double, otherwise it is interpreted as a long. That means, always add a '.' to your number if you want it intepreted as a double value. -Also, following in-built functions are supported. +Expressions can contain variables. Variable names may contain letters, digits, '\_' and '$'. Variable names must not begin with a digit. + +For logical operators, a number is true if and only if it is positive. (0 means false) + +Also, the following in-built functions are supported. |name|description| +|----|-----------| |sqrt|sqrt(x) would return square root of x| -|if|if(predicate,then,else) would return `then` if predicate evaluates to a positive number or `else` is returned| - -### How to use? - -``` -Map bindings = new HashMap<>(); -bindings.put("x", 2); - -Number result = Parser.parse("x + 2").eval(bindings); -Assert.assertEquals(4, result.longValue()); -``` +|if|if(predicate,then,else) returns 'then' if 'predicate' evaluates to a positive number, otherwise it returns 'else'| diff --git a/pom.xml b/pom.xml index b5a06fe3d95..08b7afc0359 100644 --- a/pom.xml +++ b/pom.xml @@ -444,12 +444,12 @@ org.antlr antlr4-runtime - 4.0 + 4.5.1 org.antlr antlr4-coordinator - 4.0 + 4.5.1 commons-cli @@ -740,7 +740,7 @@ org.antlr antlr4-maven-plugin - 4.0 + 4.5.1 org.apache.maven.plugins From aa6a230c9067a736bea6ab70b5f00df32da108dd Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Wed, 16 Dec 2015 10:57:04 -0600 Subject: [PATCH 3/3] remove DruidSQL.g4, its failing with newer version of ANTLR, will bring it back and fix if needed later --- server/pom.xml | 15 - .../antlr4/io/druid/sql/antlr4/DruidSQL.g4 | 343 ------------------ .../java/io/druid/server/sql/SQLRunner.java | 229 ------------ 3 files changed, 587 deletions(-) delete mode 100644 server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4 delete mode 100644 server/src/main/java/io/druid/server/sql/SQLRunner.java diff --git a/server/pom.xml b/server/pom.xml index deffcea39eb..17dc68d3108 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -132,10 +132,6 @@ org.eclipse.aether aether-api - - org.antlr - antlr4-runtime - net.spy spymemcached @@ -222,17 +218,6 @@ - - org.antlr - antlr4-maven-plugin - - - - antlr4 - - - - diff --git a/server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4 b/server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4 deleted file mode 100644 index b6e781f64ee..00000000000 --- a/server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4 +++ /dev/null @@ -1,343 +0,0 @@ -grammar DruidSQL; - -@header { -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; -import io.druid.granularity.PeriodGranularity; -import io.druid.granularity.QueryGranularity; -import io.druid.query.aggregation.AggregatorFactory; -import io.druid.query.aggregation.CountAggregatorFactory; -import io.druid.query.aggregation.DoubleSumAggregatorFactory; -import io.druid.query.aggregation.DoubleMaxAggregatorFactory; -import io.druid.query.aggregation.DoubleMinAggregatorFactory; -import io.druid.query.aggregation.PostAggregator; -import io.druid.query.aggregation.post.ArithmeticPostAggregator; -import io.druid.query.aggregation.post.ConstantPostAggregator; -import io.druid.query.aggregation.post.FieldAccessPostAggregator; -import io.druid.query.dimension.DefaultDimensionSpec; -import io.druid.query.dimension.DimensionSpec; -import io.druid.query.filter.AndDimFilter; -import io.druid.query.filter.DimFilter; -import io.druid.query.filter.NotDimFilter; -import io.druid.query.filter.OrDimFilter; -import io.druid.query.filter.RegexDimFilter; -import io.druid.query.filter.SelectorDimFilter; -import org.antlr.v4.runtime.NoViableAltException; -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.atn.ATN; -import org.antlr.v4.runtime.atn.ATNSimulator; -import org.antlr.v4.runtime.atn.ParserATNSimulator; -import org.antlr.v4.runtime.atn.PredictionContextCache; -import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.tree.ParseTreeListener; -import org.antlr.v4.runtime.tree.TerminalNode; -import org.joda.time.DateTime; -import org.joda.time.Period; - -import java.text.NumberFormat; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -} - -@parser::members { - public Map aggregators = new LinkedHashMap(); - public List postAggregators = new LinkedList(); - public DimFilter filter; - public List intervals; - public List fields = new LinkedList(); - public QueryGranularity granularity = QueryGranularity.ALL; - public Map groupByDimensions = new LinkedHashMap(); - - String dataSourceName = null; - - public String getDataSource() { - return dataSourceName; - } - - public String unescape(String quoted) { - String unquote = quoted.trim().replaceFirst("^'(.*)'\$", "\$1"); - return unquote.replace("''", "'"); - } - - AggregatorFactory evalAgg(String name, int fn) { - switch (fn) { - case SUM: return new DoubleSumAggregatorFactory("sum("+name+")", name); - case MIN: return new DoubleMinAggregatorFactory("min("+name+")", name); - case MAX: return new DoubleMaxAggregatorFactory("max("+name+")", name); - case COUNT: return new CountAggregatorFactory(name); - } - throw new IllegalArgumentException("Unknown function [" + fn + "]"); - } - - PostAggregator evalArithmeticPostAggregator(PostAggregator a, List ops, List b) { - if(b.isEmpty()) return a; - else { - int i = 0; - - PostAggregator root = a; - while(i < ops.size()) { - List list = new LinkedList(); - List names = new LinkedList(); - - names.add(root.getName()); - list.add(root); - - Token op = ops.get(i); - - while(i < ops.size() && ops.get(i).getType() == op.getType()) { - PostAggregator e = b.get(i); - list.add(e); - names.add(e.getName()); - i++; - } - - root = new ArithmeticPostAggregator("("+Joiner.on(op.getText()).join(names)+")", op.getText(), list); - } - - return root; - } - } -} - - -AND: 'and'; -OR: 'or'; -SUM: 'sum'; -MIN: 'min'; -MAX: 'max'; -COUNT: 'count'; -AS: 'as'; -OPEN: '('; -CLOSE: ')'; -STAR: '*'; -NOT: '!' ; -PLUS: '+'; -MINUS: '-'; -DIV: '/'; -COMMA: ','; -EQ: '='; -NEQ: '!='; -MATCH: '~'; -GROUP: 'group'; - -IDENT : (LETTER)(LETTER | DIGIT | '_')* ; -QUOTED_STRING : '\'' ( ESC | ~'\'' )*? '\'' ; -ESC : '\'' '\''; - -NUMBER: DIGIT*'.'?DIGIT+(EXPONENT)?; -EXPONENT: ('e') ('+'|'-')? ('0'..'9')+; -fragment DIGIT : '0'..'9'; -fragment LETTER : 'a'..'z' | 'A'..'Z'; - -LINE_COMMENT : '--' .*? '\r'? '\n' -> skip ; -COMMENT : '/*' .*? '*/' -> skip ; -WS : (' '| '\t' | '\r' '\n' | '\n' | '\r')+ -> skip; - - - -query - : select_stmt where_stmt (groupby_stmt)? - ; - -select_stmt - : 'select' e+=aliasedExpression (',' e+=aliasedExpression)* 'from' datasource { - for(AliasedExpressionContext a : $e) { - postAggregators.add(a.p); - fields.add(a.p.getName()); - } - this.dataSourceName = $datasource.text; - } - ; - -where_stmt - : 'where' f=timeAndDimFilter { - if($f.filter != null) this.filter = $f.filter; - this.intervals = Lists.newArrayList($f.interval); - } - ; - -groupby_stmt - : GROUP 'by' groupByExpression ( COMMA! groupByExpression )* - ; - -groupByExpression - : gran=granularityFn {this.granularity = $gran.granularity;} - | dim=IDENT { this.groupByDimensions.put($dim.text, new DefaultDimensionSpec($dim.text, $dim.text)); } - ; - -datasource - : IDENT - ; - -aliasedExpression returns [PostAggregator p] - : expression ( AS^ name=IDENT )? { - if($name != null) { - postAggregators.add($expression.p); - $p = new FieldAccessPostAggregator($name.text, $expression.p.getName()); - } - else $p = $expression.p; - } - ; - -expression returns [PostAggregator p] - : additiveExpression { $p = $additiveExpression.p; } - ; - -additiveExpression returns [PostAggregator p] - : a=multiplyExpression (( ops+=PLUS^ | ops+=MINUS^ ) b+=multiplyExpression)* { - List rhs = new LinkedList(); - for(MultiplyExpressionContext e : $b) rhs.add(e.p); - $p = evalArithmeticPostAggregator($a.p, $ops, rhs); - } - ; - -multiplyExpression returns [PostAggregator p] - : a=unaryExpression ((ops+= STAR | ops+=DIV ) b+=unaryExpression)* { - List rhs = new LinkedList(); - for(UnaryExpressionContext e : $b) rhs.add(e.p); - $p = evalArithmeticPostAggregator($a.p, $ops, rhs); - } - ; - -unaryExpression returns [PostAggregator p] - : MINUS e=unaryExpression { - if($e.p instanceof ConstantPostAggregator) { - ConstantPostAggregator c = (ConstantPostAggregator)$e.p; - double v = c.getConstantValue().doubleValue() * -1; - $p = new ConstantPostAggregator(Double.toString(v), v); - } else { - $p = new ArithmeticPostAggregator( - "-"+$e.p.getName(), - "*", - Lists.newArrayList($e.p, new ConstantPostAggregator("-1", -1.0)) - ); - } - } - | PLUS e=unaryExpression { $p = $e.p; } - | primaryExpression { $p = $primaryExpression.p; } - ; - -primaryExpression returns [PostAggregator p] - : constant { $p = $constant.c; } - | aggregate { - aggregators.put($aggregate.agg.getName(), $aggregate.agg); - $p = new FieldAccessPostAggregator($aggregate.agg.getName(), $aggregate.agg.getName()); - } - | OPEN! e=expression CLOSE! { $p = $e.p; } - ; - -aggregate returns [AggregatorFactory agg] - : fn=( SUM^ | MIN^ | MAX^ ) OPEN! name=(IDENT|COUNT) CLOSE! { $agg = evalAgg($name.text, $fn.type); } - | fn=COUNT OPEN! STAR CLOSE! { $agg = evalAgg("count(*)", $fn.type); } - ; - -constant returns [ConstantPostAggregator c] - : value=NUMBER { double v = Double.parseDouble($value.text); $c = new ConstantPostAggregator(Double.toString(v), v); } - ; - -/* time filters must be top level filters */ -timeAndDimFilter returns [DimFilter filter, org.joda.time.Interval interval] - : (f1=dimFilter AND)? t=timeFilter (AND f2=dimFilter)? { - if($f1.ctx != null || $f2.ctx != null) { - if($f1.ctx != null && $f2.ctx != null) { - $filter = new AndDimFilter(Lists.newArrayList($f1.filter, $f2.filter)); - } else if($f1.ctx != null) { - $filter = $f1.filter; - } else { - $filter = $f2.filter; - } - } - $interval = $t.interval; - } - ; - -dimFilter returns [DimFilter filter] - : e=orDimFilter { $filter = $e.filter; } - ; - -orDimFilter returns [DimFilter filter] - : a=andDimFilter (OR^ b+=andDimFilter)* { - if($b.isEmpty()) $filter = $a.filter; - else { - List rest = new ArrayList(); - for(AndDimFilterContext e : $b) rest.add(e.filter); - $filter = new OrDimFilter(Lists.asList($a.filter, rest.toArray(new DimFilter[]{}))); - } - } - ; - -andDimFilter returns [DimFilter filter] - : a=primaryDimFilter (AND^ b+=primaryDimFilter)* { - if($b.isEmpty()) $filter = $a.filter; - else { - List rest = new ArrayList(); - for(PrimaryDimFilterContext e : $b) rest.add(e.filter); - $filter = new AndDimFilter(Lists.asList($a.filter, rest.toArray(new DimFilter[]{}))); - } - } - ; - -primaryDimFilter returns [DimFilter filter] - : e=selectorDimFilter { $filter = $e.filter; } - | l=inListDimFilter { $filter = $l.filter; } - | NOT f=dimFilter { $filter = new NotDimFilter($f.filter); } - | OPEN! f=dimFilter CLOSE! { $filter = $f.filter; } - ; - -selectorDimFilter returns [DimFilter filter] - : dimension=IDENT op=(EQ|NEQ|MATCH) value=QUOTED_STRING { - String dim = $dimension.text; - String val = unescape($value.text); - switch($op.type) { - case(EQ): $filter = new SelectorDimFilter(dim, val, null); break; - case(NEQ): $filter = new NotDimFilter(new SelectorDimFilter(dim, val, null)); break; - case(MATCH): $filter = new RegexDimFilter(dim, val, null); break; - } - } - ; - -inListDimFilter returns [DimFilter filter] - : dimension=IDENT 'in' (OPEN! ( (list+=QUOTED_STRING (COMMA! list+=QUOTED_STRING)*) ) CLOSE!) { - List filterList = new LinkedList(); - for(Token e : $list) filterList.add(new SelectorDimFilter($dimension.text, unescape(e.getText()), null)); - $filter = new OrDimFilter(filterList); - } - ; - -timeFilter returns [org.joda.time.Interval interval, QueryGranularity granularity] - : 'timestamp' 'between' s=timestamp AND e=timestamp { - $interval = new org.joda.time.Interval($s.t, $e.t); - } - ; - -granularityFn returns [QueryGranularity granularity] - : 'granularity' OPEN! 'timestamp' ',' str=QUOTED_STRING CLOSE! { - String granStr = unescape($str.text); - try { - $granularity = QueryGranularity.fromString(granStr); - } catch(IllegalArgumentException e) { - $granularity = new PeriodGranularity(new Period(granStr), null, null); - } - } - ; - -timestamp returns [DateTime t] - : NUMBER { - String str = $NUMBER.text.trim(); - try { - $t = new DateTime(NumberFormat.getInstance().parse(str)); - } - catch(ParseException e) { - throw new IllegalArgumentException("Unable to parse number [" + str + "]"); - } - } - | QUOTED_STRING { $t = new DateTime(unescape($QUOTED_STRING.text)); } - ; diff --git a/server/src/main/java/io/druid/server/sql/SQLRunner.java b/server/src/main/java/io/druid/server/sql/SQLRunner.java deleted file mode 100644 index eda71827f0a..00000000000 --- a/server/src/main/java/io/druid/server/sql/SQLRunner.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.server.sql; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.google.common.base.Charsets; -import com.google.common.base.Function; -import com.google.common.base.Joiner; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.metamx.common.StringUtils; -import com.metamx.common.guava.CloseQuietly; -import io.druid.data.input.Row; -import io.druid.jackson.DefaultObjectMapper; -import io.druid.query.Druids; -import io.druid.query.Query; -import io.druid.query.Result; -import io.druid.query.aggregation.AggregatorFactory; -import io.druid.query.dimension.DimensionSpec; -import io.druid.query.groupby.GroupByQuery; -import io.druid.query.timeseries.TimeseriesResultValue; -import io.druid.sql.antlr4.DruidSQLLexer; -import io.druid.sql.antlr4.DruidSQLParser; -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.ConsoleErrorListener; -import org.antlr.v4.runtime.TokenStream; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.GnuParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; - -import javax.annotation.Nullable; -import javax.ws.rs.core.MediaType; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.net.URL; -import java.net.URLConnection; -import java.util.ArrayList; -import java.util.List; - -public class SQLRunner -{ - private static final String STATEMENT = "select count(*), (1 - count(*) / sum(count)) * 100 as ratio from wikipedia where" - + " timestamp between '2013-02-01' and '2013-02-14'" - + " and (namespace = 'article' or page ~ 'Talk:.*')" - + " and language in ( 'en', 'fr' ) " - + " and user ~ '(?i)^david.*'" - + " group by granularity(timestamp, 'day'), language"; - - public static void main(String[] args) throws Exception - { - - Options options = new Options(); - options.addOption("h", "help", false, "help"); - options.addOption("v", false, "verbose"); - options.addOption("e", "host", true, "endpoint [hostname:port]"); - - CommandLine cmd = new GnuParser().parse(options, args); - - if(cmd.hasOption("h")) { - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("SQLRunner", options); - System.exit(2); - } - - String hostname = cmd.getOptionValue("e", "localhost:8080"); - String sql = cmd.getArgs().length > 0 ? cmd.getArgs()[0] : STATEMENT; - - ObjectMapper objectMapper = new DefaultObjectMapper(); - ObjectWriter jsonWriter = objectMapper.writerWithDefaultPrettyPrinter(); - - CharStream stream = new ANTLRInputStream(sql); - DruidSQLLexer lexer = new DruidSQLLexer(stream); - TokenStream tokenStream = new CommonTokenStream(lexer); - DruidSQLParser parser = new DruidSQLParser(tokenStream); - lexer.removeErrorListeners(); - parser.removeErrorListeners(); - - lexer.addErrorListener(ConsoleErrorListener.INSTANCE); - parser.addErrorListener(ConsoleErrorListener.INSTANCE); - - try { - DruidSQLParser.QueryContext queryContext = parser.query(); - if(parser.getNumberOfSyntaxErrors() > 0) throw new IllegalStateException(); -// parser.setBuildParseTree(true); -// System.err.println(q.toStringTree(parser)); - } catch(Exception e) { - String msg = e.getMessage(); - if(msg != null) System.err.println(e); - System.exit(1); - } - - final Query query; - final TypeReference typeRef; - boolean groupBy = false; - if(parser.groupByDimensions.isEmpty()) { - query = Druids.newTimeseriesQueryBuilder() - .dataSource(parser.getDataSource()) - .aggregators(new ArrayList(parser.aggregators.values())) - .postAggregators(parser.postAggregators) - .intervals(parser.intervals) - .granularity(parser.granularity) - .filters(parser.filter) - .build(); - - typeRef = new TypeReference>>(){}; - } else { - query = GroupByQuery.builder() - .setDataSource(parser.getDataSource()) - .setAggregatorSpecs(new ArrayList(parser.aggregators.values())) - .setPostAggregatorSpecs(parser.postAggregators) - .setInterval(parser.intervals) - .setGranularity(parser.granularity) - .setDimFilter(parser.filter) - .setDimensions(new ArrayList(parser.groupByDimensions.values())) - .build(); - - typeRef = new TypeReference>(){}; - groupBy = true; - } - - String queryStr = jsonWriter.writeValueAsString(query); - if(cmd.hasOption("v")) System.err.println(queryStr); - - URL url = new URL(String.format("http://%s/druid/v2/?pretty", hostname)); - final URLConnection urlConnection = url.openConnection(); - urlConnection.addRequestProperty("content-type", MediaType.APPLICATION_JSON); - urlConnection.getOutputStream().write(StringUtils.toUtf8(queryStr)); - BufferedReader stdInput = new BufferedReader(new InputStreamReader(urlConnection.getInputStream(), Charsets.UTF_8)); - - Object res = objectMapper.readValue(stdInput, typeRef); - - Joiner tabJoiner = Joiner.on("\t"); - - if(groupBy) { - List rows = (List)res; - Iterable dimensions = Iterables.transform(parser.groupByDimensions.values(), new Function() - { - @Override - public String apply(@Nullable DimensionSpec input) - { - return input.getOutputName(); - } - }); - - System.out.println(tabJoiner.join(Iterables.concat( - Lists.newArrayList("timestamp"), - dimensions, - parser.fields - ))); - for(final Row r : rows) { - System.out.println( - tabJoiner.join( - Iterables.concat( - Lists.newArrayList(parser.granularity.toDateTime(r.getTimestampFromEpoch())), - Iterables.transform( - parser.groupByDimensions.values(), new Function() - { - @Override - public String apply(@Nullable DimensionSpec input) - { - return Joiner.on(",").join(r.getDimension(input.getOutputName())); - } - }), - Iterables.transform(parser.fields, new Function() - { - @Override - public Object apply(@Nullable String input) - { - return r.getFloatMetric(input); - } - }) - ) - ) - ); - } - } - else { - List> rows = (List>)res; - System.out.println(tabJoiner.join(Iterables.concat( - Lists.newArrayList("timestamp"), - parser.fields - ))); - for(final Result r : rows) { - System.out.println( - tabJoiner.join( - Iterables.concat( - Lists.newArrayList(r.getTimestamp()), - Lists.transform( - parser.fields, - new Function() - { - @Override - public Object apply(@Nullable String input) - { - return r.getValue().getMetric(input); - } - } - ) - ) - ) - ); - } - } - - CloseQuietly.close(stdInput); - } -}