From 36ccfbd20e2e73889de8d86a4a5b31a1b0ed22b7 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Mon, 14 Dec 2015 01:15:50 -0600 Subject: [PATCH] math expression language with hand written parser/lexer --- .../main/java/io/druid/math/expr/Expr.java | 398 ++++++++++++++++++ .../java/io/druid/math/expr/Function.java | 64 +++ .../main/java/io/druid/math/expr/Lexer.java | 229 ++++++++++ .../main/java/io/druid/math/expr/Parser.java | 127 ++++++ .../main/java/io/druid/math/expr/Token.java | 180 ++++++++ .../java/io/druid/math/expr/EvalTest.java | 109 +++++ .../java/io/druid/math/expr/LexerTest.java | 61 +++ .../java/io/druid/math/expr/ParserTest.java | 266 ++++++++++++ docs/content/misc/math-expr.md | 34 ++ 9 files changed, 1468 insertions(+) create mode 100644 common/src/main/java/io/druid/math/expr/Expr.java create mode 100644 common/src/main/java/io/druid/math/expr/Function.java create mode 100644 common/src/main/java/io/druid/math/expr/Lexer.java create mode 100644 common/src/main/java/io/druid/math/expr/Parser.java create mode 100644 common/src/main/java/io/druid/math/expr/Token.java create mode 100644 common/src/test/java/io/druid/math/expr/EvalTest.java create mode 100644 common/src/test/java/io/druid/math/expr/LexerTest.java create mode 100644 common/src/test/java/io/druid/math/expr/ParserTest.java create mode 100644 docs/content/misc/math-expr.md diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java new file mode 100644 index 00000000000..7e24c11d25f --- /dev/null +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -0,0 +1,398 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +import com.google.common.math.LongMath; + +import java.util.List; +import java.util.Map; + +/** + */ +public interface Expr +{ + Number eval(Map bindings); +} + +class SimpleExpr implements Expr +{ + private final Atom atom; + + public SimpleExpr(Atom atom) + { + this.atom = atom; + } + + @Override + public String toString() + { + return atom.toString(); + } + + @Override + public Number eval(Map bindings) + { + return atom.eval(bindings); + } +} + +class BinExpr implements Expr +{ + private final Token opToken; + private final Expr left; + private final Expr right; + + public BinExpr(Token opToken, Expr left, Expr right) + { + this.opToken = opToken; + this.left = left; + this.right = right; + } + + public Number eval(Map bindings) + { + switch(opToken.getType()) { + case Token.AND: + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + long lval = leftVal.longValue(); + if (lval > 0) { + long rval = rightVal.longValue(); + return rval > 0 ? 1 : 0; + } else { + return 0; + } + } else { + double lval = leftVal.doubleValue(); + if (lval > 0) { + double rval = rightVal.doubleValue(); + return rval > 0 ? 1.0d : 0.0d; + } else { + return 0.0d; + } + } + case Token.OR: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + long lval = leftVal.longValue(); + if (lval > 0) { + return 1; + } else { + long rval = rightVal.longValue(); + return rval > 0 ? 1 : 0; + } + } else { + double lval = leftVal.doubleValue(); + if (lval > 0) { + return 1.0d; + } else { + double rval = rightVal.doubleValue(); + return rval > 0 ? 1.0d : 0.0d; + } + } + case Token.LT: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() < rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() < rightVal.doubleValue() ? 1.0d : 0.0d; + } + case Token.LEQ: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() <= rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() <= rightVal.doubleValue() ? 1.0d : 0.0d; + } + case Token.GT: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() > rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() > rightVal.doubleValue() ? 1.0d : 0.0d; + } + case Token.GEQ: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() >= rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() >= rightVal.doubleValue() ? 1.0d : 0.0d; + } + case Token.EQ: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() == rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() == rightVal.doubleValue() ? 1.0d : 0.0d; + } + case Token.NEQ: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() != rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() != rightVal.doubleValue() ? 1.0d : 0.0d; + } + case Token.PLUS: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() + rightVal.longValue(); + } else { + return leftVal.doubleValue() + rightVal.doubleValue(); + } + case Token.MINUS: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() - rightVal.longValue(); + } else { + return leftVal.doubleValue() - rightVal.doubleValue(); + } + case Token.MUL: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() * rightVal.longValue(); + } else { + return leftVal.doubleValue() * rightVal.doubleValue(); + } + case Token.DIV: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() / rightVal.longValue(); + } else { + return leftVal.doubleValue() / rightVal.doubleValue(); + } + case Token.MODULO: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() % rightVal.longValue(); + } else { + return leftVal.doubleValue() % rightVal.doubleValue(); + } + case Token.CARROT: + leftVal = left.eval(bindings); + rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return LongMath.pow(leftVal.longValue(), rightVal.intValue()); + } else { + return Math.pow(leftVal.doubleValue(), rightVal.doubleValue()); + } + default: + throw new RuntimeException("Unknown operator " + opToken.getMatch()); + } + } + + private boolean isLong(Number left, Number right) + { + return left instanceof Long && right instanceof Long; + } + + @Override + public String toString() + { + return "(" + opToken.getMatch() + " " + left + " " + right + ")"; + } +} + + + +interface Atom +{ + Number eval(Map bindings); +} + +class LongValueAtom implements Atom +{ + private final long value; + + public LongValueAtom(long value) + { + this.value = value; + } + + @Override + public String toString() + { + return String.valueOf(value); + } + + @Override + public Number eval(Map bindings) + { + return value; + } +} + +class DoubleValueAtom implements Atom +{ + private final double value; + + public DoubleValueAtom(double value) + { + this.value = value; + } + + @Override + public String toString() + { + return String.valueOf(value); + } + + @Override + public Number eval(Map bindings) + { + return value; + } +} + +class IdentifierAtom implements Atom +{ + private final Token value; + + public IdentifierAtom(Token value) + { + this.value = value; + } + + @Override + public String toString() + { + return value.getMatch(); + } + + @Override + public Number eval(Map bindings) + { + Number val = bindings.get(value.getMatch()); + if (val == null) { + throw new RuntimeException("No binding found for " + value.getMatch()); + } else { + return val; + } + } +} + +class UnaryNotExprAtom implements Atom +{ + private final Expr expr; + + public UnaryNotExprAtom(Expr expr) + { + this.expr = expr; + } + + @Override + public String toString() + { + return "!" + expr.toString(); + } + + @Override + public Number eval(Map bindings) + { + Number valObj = expr.eval(bindings); + return valObj.doubleValue() > 0 ? 0.0d : 1.0d; + } +} + +class UnaryMinusExprAtom implements Atom +{ + private final Expr expr; + + public UnaryMinusExprAtom(Expr expr) + { + this.expr = expr; + } + + @Override + public String toString() + { + return "-" + expr.toString(); + } + + @Override + public Number eval(Map bindings) + { + Number valObj = expr.eval(bindings); + if (valObj instanceof Long) { + return -1 * valObj.longValue(); + } else { + return -1 * valObj.doubleValue(); + } + } +} + + +class NestedExprAtom implements Atom +{ + private final Expr expr; + + public NestedExprAtom(Expr expr) + { + this.expr = expr; + } + + @Override + public String toString() + { + return expr.toString(); + } + + @Override + public Number eval(Map bindings) + { + return expr.eval(bindings); + } +} + +class FunctionAtom implements Atom +{ + private final String name; + private final List args; + + public FunctionAtom(String name, List args) + { + this.name = name; + this.args = args; + } + + @Override + public String toString() + { + return "(" + name + " " + args + ")"; + } + + @Override + public Number eval(Map bindings) + { + return Parser.func.get(name).apply(args, bindings); + } +} diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java new file mode 100644 index 00000000000..630642ba867 --- /dev/null +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -0,0 +1,64 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +import java.util.List; +import java.util.Map; + +/** + */ +interface Function +{ + Number apply(List args, Map bindings); +} + +class SqrtFunc implements Function +{ + + @Override + public Number apply(List args, Map bindings) + { + if (args.size() != 1) { + throw new RuntimeException("function 'sqrt' needs 1 argument"); + } + + Number x = args.get(0).eval(bindings); + return Math.sqrt(x.doubleValue()); + } +} + +class ConditionFunc implements Function +{ + + @Override + public Number apply(List args, Map bindings) + { + if (args.size() != 3) { + throw new RuntimeException("function 'if' needs 3 argument"); + } + + Number x = args.get(0).eval(bindings); + if (x instanceof Long) { + return x.longValue() > 0 ? args.get(1).eval(bindings) : args.get(2).eval(bindings); + } else { + return x.doubleValue() > 0 ? args.get(1).eval(bindings) : args.get(2).eval(bindings); + } + } +} diff --git a/common/src/main/java/io/druid/math/expr/Lexer.java b/common/src/main/java/io/druid/math/expr/Lexer.java new file mode 100644 index 00000000000..fc55a689ea6 --- /dev/null +++ b/common/src/main/java/io/druid/math/expr/Lexer.java @@ -0,0 +1,229 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +class Lexer +{ + private String input; + private int currPos; + + private Token next; + + Lexer(String input) + { + this.input = input; + currPos = 0; + + next = nextToken(); + } + + Token peek() + { + return next; + } + + Token consume() + { + Token old = next; + next = nextToken(); + return old; + } + + private Token nextToken() + { + if (currPos >= input.length()) { + return new Token(Token.EOF); + } + + char c = input.charAt(currPos); + while(c == ' ') { + currPos++; + if (currPos >= input.length()) { + return new Token(Token.EOF); + } + c = input.charAt(currPos); + } + + switch(c) + { + case '<': + currPos++; + if (currPos < input.length() && input.charAt(currPos) == '=') { + currPos++; + return new Token(Token.LEQ, "<="); + } else { + return new Token(Token.LT, "<"); + } + case '>': + currPos++; + if (currPos < input.length() && input.charAt(currPos) == '=') { + currPos++; + return new Token(Token.GEQ, ">="); + } else { + return new Token(Token.GT, ">"); + } + case '=': + currPos++; + if (currPos < input.length() && input.charAt(currPos) == '=') { + currPos++; + return new Token(Token.EQ, "=="); + } else { + throw new IllegalArgumentException("unknown operator '='"); + } + case '!': + currPos++; + if (currPos < input.length() && input.charAt(currPos) == '=') { + currPos++; + return new Token(Token.NEQ, "!="); + } else { + return new Token(Token.NOT, "!"); + } + + case '+': + currPos++; + return new Token(Token.PLUS, "+"); + case '-': + currPos++; + return new Token(Token.MINUS, "-"); + + case '*': + currPos++; + return new Token(Token.MUL, "*"); + case '/': + currPos++; + return new Token(Token.DIV, "/"); + case '%': + currPos++; + return new Token(Token.MODULO, "%"); + + case '^': + currPos++; + return new Token(Token.CARROT, "^"); + + case '(': + currPos++; + return new Token(Token.LPAREN, "("); + case ')': + currPos++; + return new Token(Token.RPAREN, ")"); + case ',': + currPos++; + return new Token(Token.COMMA, ","); + + default: + if (isNumberStartingChar(c)) { + return parseNumber(); + } else if (isIdentifierStartingChar(c)){ + return parseIdentifierOrKeyword(); + } else { + throw new RuntimeException("Illegal expression " + toString()); + } + } + } + + private boolean isNumberStartingChar(char c) + { + return c >= '0' && c <= '9'; + } + + private boolean isIdentifierStartingChar(char c) + { + return (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + c == '$' || + c == '_'; + } + + private boolean isIdentifierChar(char c) + { + return (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '$' || + c == '_'; + } + + private Token parseIdentifierOrKeyword() + { + StringBuilder sb = new StringBuilder(); + char c = input.charAt(currPos++); + while (isIdentifierChar(c)) { + sb.append(c); + + if (currPos < input.length()) { + c = input.charAt(currPos++); + } else { + currPos++; + break; + } + }; + + currPos--; + + String str = sb.toString(); + if(str.equals("and")) { + return new Token(Token.AND, str); + } else if(str.equals("or")) { + return new Token(Token.OR, str); + } else { + return new Token(Token.IDENTIFIER, sb.toString()); + } + } + + // Numbers + // long : [0-9]+ + // double : [0-9]+.[0-9]+ + private Token parseNumber() + { + boolean isLong = true; + StringBuilder sb = new StringBuilder(); + char c = input.charAt(currPos++); + + while ( + ('0' <= c && c <= '9') || c == '.' + ) { + if (c == '.') { + isLong = false; + } + + sb.append(c); + + if (currPos < input.length()) { + c = input.charAt(currPos++); + } else { + currPos++; + break; + } + }; + + currPos--; + if (isLong) { + return new Token(Token.LONG, sb.toString()); + } else { + return new Token(Token.DOUBLE, sb.toString()); + } + } + + @Override + public String toString() + { + return "at " + currPos + ", " + input; + } +} diff --git a/common/src/main/java/io/druid/math/expr/Parser.java b/common/src/main/java/io/druid/math/expr/Parser.java new file mode 100644 index 00000000000..588db098f45 --- /dev/null +++ b/common/src/main/java/io/druid/math/expr/Parser.java @@ -0,0 +1,127 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Parser +{ + static final Map func = new HashMap<>(); + + static { + func.put("sqrt", new SqrtFunc()); + func.put("if", new ConditionFunc()); + } + + private final Lexer lexer; + + private Parser(Lexer lexer) + { + this.lexer = lexer; + } + + public static Expr parse(String input) + { + return new Parser(new Lexer(input)).parseExpr(0); + } + + private Expr parseExpr(int p) + { + Expr result = new SimpleExpr(parseAtom()); + + Token t = lexer.peek(); + while ( + t.getType() < Token.NUM_BINARY_OPERATORS + && + Token.PRECEDENCE[t.getType()] >= p + ) { + lexer.consume(); + result = new BinExpr( + t, + result, + parseExpr(Token.R_PRECEDENCE[t.getType()]) + ); + + t = lexer.peek(); + } + + return result; + } + + private Atom parseAtom() + { + Token t = lexer.peek(); + + switch(t.getType()) { + case Token.IDENTIFIER: + lexer.consume(); + String id = t.getMatch(); + + if (func.containsKey(id)) { + expect(Token.LPAREN); + List args = new ArrayList<>(); + t = lexer.peek(); + while(t.getType() != Token.RPAREN) { + args.add(parseExpr(0)); + t = lexer.peek(); + if (t.getType() == Token.COMMA) { + lexer.consume(); + } + } + expect(Token.RPAREN); + return new FunctionAtom(id, args); + } + + return new IdentifierAtom(t); + case Token.LONG: + lexer.consume(); + return new LongValueAtom(Long.valueOf(t.getMatch())); + case Token.DOUBLE: + lexer.consume(); + return new DoubleValueAtom(Double.valueOf(t.getMatch())); + case Token.MINUS: + lexer.consume(); + return new UnaryMinusExprAtom(parseExpr(Token.UNARY_MINUS_PRECEDENCE)); + case Token.NOT: + lexer.consume(); + return new UnaryNotExprAtom(parseExpr(Token.UNARY_NOT_PRECEDENCE)); + case Token.LPAREN: + lexer.consume(); + Expr expression = parseExpr(0); + if(lexer.consume().getType() == Token.RPAREN) { + return new NestedExprAtom(expression); + } + default: + throw new RuntimeException("Invalid token found " + t + " in input " + lexer); + } + } + + private void expect(int type) + { + Token t = lexer.consume(); + if(t.getType() != type) { + throw new RuntimeException("Invalid token found " + t + " in input " + lexer); + } + } +} diff --git a/common/src/main/java/io/druid/math/expr/Token.java b/common/src/main/java/io/druid/math/expr/Token.java new file mode 100644 index 00000000000..25cf6ac3e34 --- /dev/null +++ b/common/src/main/java/io/druid/math/expr/Token.java @@ -0,0 +1,180 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +/** + */ +class Token +{ + static final int AND = 0; + static final int OR = 1; + + static final int LT = 2; + static final int LEQ = 3; + static final int GT = 4; + static final int GEQ = 5; + static final int EQ = 6; + static final int NEQ = 7; + + static final int PLUS = 8; + static final int MINUS = 9; + + static final int MUL = 10; + static final int DIV = 11; + static final int MODULO = 12; + + static final int CARROT = 13; + static final int NUM_BINARY_OPERATORS = 14; + + static final int LPAREN = 51; + static final int RPAREN = 52; + static final int COMMA = 53; + static final int NOT = 54; + static final int IDENTIFIER = 55; + static final int LONG = 56; + static final int DOUBLE = 57; + + static final int EOF = 100; + + static final int[] PRECEDENCE; + static final int[] R_PRECEDENCE; + static final int UNARY_MINUS_PRECEDENCE; + static final int UNARY_NOT_PRECEDENCE; + + static { + PRECEDENCE = new int[NUM_BINARY_OPERATORS]; + R_PRECEDENCE = new int[NUM_BINARY_OPERATORS]; + //R_RECEDENCE = (op is left-associative) ? PRECEDENCE + 1 : PRECEDENCE + + int precedenceCounter = 0; + + PRECEDENCE[AND] = precedenceCounter; + R_PRECEDENCE[AND] = PRECEDENCE[AND] + 1; + + PRECEDENCE[OR] = precedenceCounter; + R_PRECEDENCE[OR] = PRECEDENCE[OR] + 1; + + precedenceCounter++; + + PRECEDENCE[EQ] = precedenceCounter; + R_PRECEDENCE[EQ] = PRECEDENCE[EQ] + 1; + + PRECEDENCE[NEQ] = precedenceCounter; + R_PRECEDENCE[NEQ] = PRECEDENCE[NEQ] + 1; + + PRECEDENCE[LT] = precedenceCounter; + R_PRECEDENCE[LT] = PRECEDENCE[LT] + 1; + + PRECEDENCE[LEQ] = precedenceCounter; + R_PRECEDENCE[LEQ] = PRECEDENCE[LEQ] + 1; + + PRECEDENCE[GT] = precedenceCounter; + R_PRECEDENCE[GT] = PRECEDENCE[GT] + 1; + + PRECEDENCE[GEQ] = precedenceCounter; + R_PRECEDENCE[GEQ] = PRECEDENCE[GEQ] + 1; + + precedenceCounter++; + + PRECEDENCE[PLUS] = precedenceCounter; + R_PRECEDENCE[PLUS] = PRECEDENCE[PLUS] + 1; + + PRECEDENCE[MINUS] = precedenceCounter; + R_PRECEDENCE[MINUS] = PRECEDENCE[MINUS] + 1; + + precedenceCounter++; + + PRECEDENCE[MUL] = precedenceCounter; + R_PRECEDENCE[MUL] = PRECEDENCE[MUL] + 1; + + PRECEDENCE[DIV] = precedenceCounter; + R_PRECEDENCE[DIV] = PRECEDENCE[DIV] + 1; + + PRECEDENCE[MODULO] = precedenceCounter; + R_PRECEDENCE[MODULO] = PRECEDENCE[MODULO] + 1; + + precedenceCounter++; + + PRECEDENCE[CARROT] = precedenceCounter; + R_PRECEDENCE[CARROT] = PRECEDENCE[CARROT]; + + precedenceCounter++; + + UNARY_MINUS_PRECEDENCE = precedenceCounter; + UNARY_NOT_PRECEDENCE = precedenceCounter; + } + + private final int type; + private final String match; + + Token(int type) + { + this(type, ""); + } + + Token(int type, String match) + { + this.type = type; + this.match = match; + } + + int getType() + { + return type; + } + + String getMatch() + { + return match; + } + + @Override + public String toString() + { + return match; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + Token token = (Token) o; + + if (type != token.type) { + return false; + } + return !(match != null ? !match.equals(token.match) : token.match != null); + + } + + @Override + public int hashCode() + { + int result = type; + result = 31 * result + (match != null ? match.hashCode() : 0); + return result; + } +} diff --git a/common/src/test/java/io/druid/math/expr/EvalTest.java b/common/src/test/java/io/druid/math/expr/EvalTest.java new file mode 100644 index 00000000000..81976b5c138 --- /dev/null +++ b/common/src/test/java/io/druid/math/expr/EvalTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +/** + */ +public class EvalTest +{ + @Test + public void testDoubleEval() + { + Map bindings = new HashMap<>(); + bindings.put("x", 2.0d); + + Assert.assertEquals(2.0, Parser.parse("x").eval(bindings).doubleValue(), 0.0001); + + Assert.assertFalse(Parser.parse("1.0 and 0.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("1.0 and 2.0").eval(bindings).doubleValue() > 0.0); + + Assert.assertTrue(Parser.parse("1.0 or 0.0").eval(bindings).doubleValue() > 0.0); + Assert.assertFalse(Parser.parse("0.0 or 0.0").eval(bindings).doubleValue() > 0.0); + + Assert.assertTrue(Parser.parse("2.0 > 1.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("2.0 >= 2.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("1.0 < 2.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("2.0 <= 2.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("2.0 == 2.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("2.0 != 1.0").eval(bindings).doubleValue() > 0.0); + + Assert.assertEquals(3.5, Parser.parse("2.0 + 1.5").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(0.5, Parser.parse("2.0 - 1.5").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(3.0, Parser.parse("2.0 * 1.5").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(4.0, Parser.parse("2.0 / 0.5").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(0.2, Parser.parse("2.0 % 0.3").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(8.0, Parser.parse("2.0 ^ 3.0").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(-1.5, Parser.parse("-1.5").eval(bindings).doubleValue(), 0.0001); + + Assert.assertTrue(Parser.parse("!-1.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("!0.0").eval(bindings).doubleValue() > 0.0); + Assert.assertFalse(Parser.parse("!2.0").eval(bindings).doubleValue() > 0.0); + + Assert.assertEquals(2.0, Parser.parse("sqrt(4.0)").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(2.0, Parser.parse("if(1.0, 2.0, 3.0)").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(3.0, Parser.parse("if(0.0, 2.0, 3.0)").eval(bindings).doubleValue(), 0.0001); + } + + @Test + public void testLongEval() + { + Map bindings = new HashMap<>(); + bindings.put("x", 9223372036854775807l); + + Assert.assertEquals(9223372036854775807l, Parser.parse("x").eval(bindings).longValue()); + + Assert.assertFalse(Parser.parse("9223372036854775807 and 0").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775807 and 9223372036854775806").eval(bindings).longValue() > 0); + + Assert.assertTrue(Parser.parse("9223372036854775807 or 0").eval(bindings).longValue() > 0); + Assert.assertFalse(Parser.parse("-9223372036854775807 or -9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("-9223372036854775807 or 9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertFalse(Parser.parse("0 or 0").eval(bindings).longValue() > 0); + + Assert.assertTrue(Parser.parse("9223372036854775807 > 9223372036854775806").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775807 >= 9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775806 < 9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775807 <= 9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775807 == 9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775807 != 9223372036854775806").eval(bindings).longValue() > 0); + + Assert.assertEquals(9223372036854775807l, Parser.parse("9223372036854775806 + 1").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775806l, Parser.parse("9223372036854775807 - 1").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775806l, Parser.parse("4611686018427387903 * 2").eval(bindings).longValue()); + Assert.assertEquals(4611686018427387903l, Parser.parse("9223372036854775806 / 2").eval(bindings).longValue()); + Assert.assertEquals(7, Parser.parse("9223372036854775807 % 9223372036854775800").eval(bindings).longValue()); + Assert.assertEquals( 9223372030926249001l, Parser.parse("3037000499 ^ 2").eval(bindings).longValue()); + Assert.assertEquals(-9223372036854775807l, Parser.parse("-9223372036854775807").eval(bindings).longValue()); + + Assert.assertTrue(Parser.parse("!-9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("!0").eval(bindings).longValue() > 0); + Assert.assertFalse(Parser.parse("!9223372036854775807").eval(bindings).longValue() > 0); + + Assert.assertEquals(3037000499l, Parser.parse("sqrt(9223372036854775807)").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775807l, Parser.parse("if(9223372036854775807, 9223372036854775807, 9223372036854775806)").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775806l, Parser.parse("if(0, 9223372036854775807, 9223372036854775806)").eval(bindings).longValue()); + } +} diff --git a/common/src/test/java/io/druid/math/expr/LexerTest.java b/common/src/test/java/io/druid/math/expr/LexerTest.java new file mode 100644 index 00000000000..01d081348dd --- /dev/null +++ b/common/src/test/java/io/druid/math/expr/LexerTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +import org.junit.Assert; +import org.junit.Test; + +/** + */ +public class LexerTest +{ + @Test + public void testAllTokens() + { + testAllTokensAsserts("abcd<<=>>===!!=+-*/%^(),123 01.23"); + testAllTokensAsserts(" abcd < <= > >= == ! !=+ -*/ %^ () , 123 01.23 "); + } + + private void testAllTokensAsserts(String input) + { + Lexer lexer = new Lexer(input); + Assert.assertEquals(new Token(Token.IDENTIFIER, "abcd"), lexer.consume()); + Assert.assertEquals(new Token(Token.LT, "<"), lexer.consume()); + Assert.assertEquals(new Token(Token.LEQ, "<="), lexer.consume()); + Assert.assertEquals(new Token(Token.GT, ">"), lexer.consume()); + Assert.assertEquals(new Token(Token.GEQ, ">="), lexer.consume()); + Assert.assertEquals(new Token(Token.EQ, "=="), lexer.consume()); + Assert.assertEquals(new Token(Token.NOT, "!"), lexer.consume()); + Assert.assertEquals(new Token(Token.NEQ, "!="), lexer.consume()); + Assert.assertEquals(new Token(Token.PLUS, "+"), lexer.consume()); + Assert.assertEquals(new Token(Token.MINUS, "-"), lexer.consume()); + Assert.assertEquals(new Token(Token.MUL, "*"), lexer.consume()); + Assert.assertEquals(new Token(Token.DIV, "/"), lexer.consume()); + Assert.assertEquals(new Token(Token.MODULO, "%"), lexer.consume()); + Assert.assertEquals(new Token(Token.CARROT, "^"), lexer.consume()); + Assert.assertEquals(new Token(Token.LPAREN, "("), lexer.consume()); + Assert.assertEquals(new Token(Token.RPAREN, ")"), lexer.consume()); + Assert.assertEquals(new Token(Token.COMMA, ","), lexer.consume()); + Assert.assertEquals(new Token(Token.LONG, "123"), lexer.consume()); + Assert.assertEquals(new Token(Token.DOUBLE, "01.23"), lexer.consume()); + Assert.assertEquals(new Token(Token.EOF, ""), lexer.consume()); + Assert.assertEquals(new Token(Token.EOF, ""), lexer.consume()); + } +} diff --git a/common/src/test/java/io/druid/math/expr/ParserTest.java b/common/src/test/java/io/druid/math/expr/ParserTest.java new file mode 100644 index 00000000000..9d52fb1f0ee --- /dev/null +++ b/common/src/test/java/io/druid/math/expr/ParserTest.java @@ -0,0 +1,266 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +import org.junit.Assert; +import org.junit.Test; + +/** + */ +public class ParserTest +{ + @Test + public void testSimple() + { + String actual = Parser.parse("1").toString(); + String expected = "1"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleUnaryOps1() + { + String actual = Parser.parse("-x").toString(); + String expected = "-x"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("!x").toString(); + expected = "!x"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleUnaryOps2() + { + String actual = Parser.parse("-1").toString(); + String expected = "-1"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("--1").toString(); + expected = "--1"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("-1+2").toString(); + expected = "(+ -1 2)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("-1*2").toString(); + expected = "(* -1 2)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("-1^2").toString(); + expected = "(^ -1 2)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleLogicalOps1() + { + String actual = Parser.parse("x>y").toString(); + String expected = "(> x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x=y").toString(); + expected = "(>= x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x==y").toString(); + expected = "(== x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x!=y").toString(); + expected = "(!= x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x and y").toString(); + expected = "(and x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x or y").toString(); + expected = "(or x y)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleAdditivityOp1() + { + String actual = Parser.parse("x+y").toString(); + String expected = "(+ x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x-y").toString(); + expected = "(- x y)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleAdditivityOp2() + { + String actual = Parser.parse("x+y+z").toString(); + String expected = "(+ (+ x y) z)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x+y-z").toString(); + expected = "(- (+ x y) z)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x-y+z").toString(); + expected = "(+ (- x y) z)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x-y-z").toString(); + expected = "(- (- x y) z)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleMultiplicativeOp1() + { + String actual = Parser.parse("x*y").toString(); + String expected = "(* x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x/y").toString(); + expected = "(/ x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x%y").toString(); + expected = "(% x y)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleMultiplicativeOp2() + { + String actual = Parser.parse("1*2*3").toString(); + String expected = "(* (* 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1*2/3").toString(); + expected = "(/ (* 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1/2*3").toString(); + expected = "(* (/ 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1/2/3").toString(); + expected = "(/ (/ 1 2) 3)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleCarrot1() + { + String actual = Parser.parse("1^2").toString(); + String expected = "(^ 1 2)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleCarrot2() + { + String actual = Parser.parse("1^2^3").toString(); + String expected = "(^ 1 (^ 2 3))"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testMixed() + { + String actual = Parser.parse("1+2*3").toString(); + String expected = "(+ 1 (* 2 3))"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1+(2*3)").toString(); + Assert.assertEquals(expected, actual); + + actual = Parser.parse("(1+2)*3").toString(); + expected = "(* (+ 1 2) 3)"; + Assert.assertEquals(expected, actual); + + + actual = Parser.parse("1*2+3").toString(); + expected = "(+ (* 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("(1*2)+3").toString(); + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1*(2+3)").toString(); + expected = "(* 1 (+ 2 3))"; + Assert.assertEquals(expected, actual); + + + actual = Parser.parse("1+2^3)").toString(); + expected = "(+ 1 (^ 2 3))"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1+(2^3))").toString(); + expected = "(+ 1 (^ 2 3))"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("(1+2)^3)").toString(); + expected = "(^ (+ 1 2) 3)"; + Assert.assertEquals(expected, actual); + + + actual = Parser.parse("1^2+3)").toString(); + expected = "(+ (^ 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("(1^2)+3").toString(); + expected = "(+ (^ 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1^(2+3)").toString(); + expected = "(^ 1 (+ 2 3))"; + Assert.assertEquals(expected, actual); + + + actual = Parser.parse("1^2*3+4)").toString(); + expected = "(+ (* (^ 1 2) 3) 4)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("-1^-2*-3+-4)").toString(); + expected = "(+ (* (^ -1 -2) -3) -4)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testFunctions() + { + String actual = Parser.parse("sqrt(x)").toString(); + String expected = "(sqrt [x])"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("if(cond,then,else)").toString(); + expected = "(if [cond, then, else])"; + Assert.assertEquals(expected, actual); + } +} diff --git a/docs/content/misc/math-expr.md b/docs/content/misc/math-expr.md new file mode 100644 index 00000000000..c993e2a5276 --- /dev/null +++ b/docs/content/misc/math-expr.md @@ -0,0 +1,34 @@ +--- +layout: doc_page +--- + +This expression language supports following operators (listed in increasing order of precedence). + +|Operators|Description| +|---------|-----------| +|and,or|Binary Logical AND, OR| +|<, <=, >, >=, ==, !=|Binary Comparison| +|+, -|Binary additive| +|*, /, %|Binary multiplicative| +|^|Binary power op| +|!, -|Unary NOT and Minus| + +long and double data types are supported, number containing a dot is interpreted as a double or else a long. That means, always add a '.' to your number if you want it intepreted as a double value. +You can use variables inside the expression which must start with an alphabet or '_' or '$' and can contain a digit as well in subsequent characters. For logical operators, positive number is considered a boolean true and false otherwise. + +Also, following in-built functions are supported. + +|name|description| +|sqrt|sqrt(x) would return square root of x| +|if|if(predicate,then,else) would return `then` if predicate evaluates to a positive number or `else` is returned| + +### How to use? + +``` +Map bindings = new HashMap<>(); +bindings.put("x", 2); + +Number result = Parser.parse("x + 2").eval(bindings); +Assert.assertEquals(4, result.longValue()); +``` +