From 4ca5fe0f60ea90e63c07d5f2e7cc30c5187878d1 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Fri, 22 Mar 2019 15:40:10 -0700 Subject: [PATCH] SQL: Add PARSE_LONG function. (#7326) * SQL: Add PARSE_LONG function. * Fix test. --- .../org/apache/druid/math/expr/Function.java | 42 +++++++++++++ docs/content/misc/math-expr.md | 1 + docs/content/querying/sql.md | 1 + .../builtin/ParseLongOperatorConversion.java | 61 +++++++++++++++++++ .../calcite/planner/DruidOperatorTable.java | 2 + .../calcite/expression/ExpressionsTest.java | 51 ++++++++++++++++ 6 files changed, 158 insertions(+) create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ParseLongOperatorConversion.java diff --git a/core/src/main/java/org/apache/druid/math/expr/Function.java b/core/src/main/java/org/apache/druid/math/expr/Function.java index 2b03bfc1038..de50cfb58d9 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Function.java +++ b/core/src/main/java/org/apache/druid/math/expr/Function.java @@ -122,6 +122,48 @@ interface Function } } + class ParseLong implements Function + { + @Override + public String name() + { + return "parse_long"; + } + + @Override + public ExprEval apply(List args, Expr.ObjectBinding bindings) + { + final int radix; + if (args.size() == 1) { + radix = 10; + } else if (args.size() == 2) { + radix = args.get(1).eval(bindings).asInt(); + } else { + throw new IAE("Function[%s] needs 1 or 2 arguments", name()); + } + + final String input = NullHandling.nullToEmptyIfNeeded(args.get(0).eval(bindings).asString()); + if (input == null) { + return ExprEval.ofLong(null); + } + + final long retVal; + try { + if (radix == 16 && (input.startsWith("0x") || input.startsWith("0X"))) { + // Strip leading 0x from hex strings. + retVal = Long.parseLong(input.substring(2), radix); + } else { + retVal = Long.parseLong(input, radix); + } + } + catch (NumberFormatException e) { + return ExprEval.ofLong(null); + } + + return ExprEval.of(retVal); + } + } + class Pi implements Function { private static final double PI = Math.PI; diff --git a/docs/content/misc/math-expr.md b/docs/content/misc/math-expr.md index a1166828c86..124935e1e84 100644 --- a/docs/content/misc/math-expr.md +++ b/docs/content/misc/math-expr.md @@ -69,6 +69,7 @@ The following built-in functions are available. |concat|concatenate a list of strings| |like|like(expr, pattern[, escape]) is equivalent to SQL `expr LIKE pattern`| |lookup|lookup(expr, lookup-name) looks up expr in a registered [query-time lookup](../querying/lookups.html)| +|parse_long|parse_long(string[, radix]) parses a string as a long with the given radix, or 10 (decimal) if a radix is not provided.| |regexp_extract|regexp_extract(expr, pattern[, index]) applies a regular expression pattern and extracts a capture group index, or null if there is no match. If index is unspecified or zero, returns the substring that matched the pattern.| |replace|replace(expr, pattern, replacement) replaces pattern with replacement| |substring|substring(expr, index, length) behaves like java.lang.String's substring| diff --git a/docs/content/querying/sql.md b/docs/content/querying/sql.md index 78b37683bb0..868ad64ca95 100644 --- a/docs/content/querying/sql.md +++ b/docs/content/querying/sql.md @@ -178,6 +178,7 @@ String functions accept strings, and return a type appropriate to the function. |`STRLEN(expr)`|Synonym for `LENGTH`.| |`LOOKUP(expr, lookupName)`|Look up expr in a registered [query-time lookup table](lookups.html).| |`LOWER(expr)`|Returns expr in all lowercase.| +|`PARSE_LONG(string[, radix])`|Parses a string into a long (BIGINT) with the given radix, or 10 (decimal) if a radix is not provided.| |`POSITION(needle IN haystack [FROM fromIndex])`|Returns the index of needle within haystack, with indexes starting from 1. The search will begin at fromIndex, or 1 if fromIndex is not specified. If the needle is not found, returns 0.| |`REGEXP_EXTRACT(expr, pattern, [index])`|Apply regular expression pattern and extract a capture group, or null if there is no match. If index is unspecified or zero, returns the substring that matched the pattern.| |`REPLACE(expr, pattern, replacement)`|Replaces pattern with replacement in expr, and returns the result.| diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ParseLongOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ParseLongOperatorConversion.java new file mode 100644 index 00000000000..3230f684052 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ParseLongOperatorConversion.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.expression.builtin; + +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.druid.sql.calcite.expression.DruidExpression; +import org.apache.druid.sql.calcite.expression.OperatorConversions; +import org.apache.druid.sql.calcite.expression.SqlOperatorConversion; +import org.apache.druid.sql.calcite.planner.PlannerContext; +import org.apache.druid.sql.calcite.table.RowSignature; + +public class ParseLongOperatorConversion implements SqlOperatorConversion +{ + private static final String NAME = "PARSE_LONG"; + + private static final SqlFunction SQL_FUNCTION = OperatorConversions + .operatorBuilder(NAME) + .operandTypes(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER) + .returnType(SqlTypeName.BIGINT) + .functionCategory(SqlFunctionCategory.STRING) + .requiredOperands(1) + .build(); + + @Override + public SqlOperator calciteOperator() + { + return SQL_FUNCTION; + } + + @Override + public DruidExpression toDruidExpression( + final PlannerContext plannerContext, + final RowSignature rowSignature, + final RexNode rexNode + ) + { + return OperatorConversions.convertCall(plannerContext, rowSignature, rexNode, "parse_long"); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java index b355095da74..c908a2d5266 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java @@ -56,6 +56,7 @@ import org.apache.druid.sql.calcite.expression.builtin.FloorOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.LTrimOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.LikeOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.MillisToTimestampOperatorConversion; +import org.apache.druid.sql.calcite.expression.builtin.ParseLongOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.PositionOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.RTrimOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.RegexpExtractOperatorConversion; @@ -172,6 +173,7 @@ public class DruidOperatorTable implements SqlOperatorTable .add(new PositionOperatorConversion()) .add(new RegexpExtractOperatorConversion()) .add(new RTrimOperatorConversion()) + .add(new ParseLongOperatorConversion()) .add(new StrposOperatorConversion()) .add(new SubstringOperatorConversion()) .add(new AliasedOperatorConversion(new SubstringOperatorConversion(), "SUBSTR")) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java index 603796dc3cb..5e68984a604 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java @@ -41,6 +41,7 @@ import org.apache.druid.math.expr.Parser; import org.apache.druid.query.extraction.RegexDimExtractionFn; import org.apache.druid.segment.column.ValueType; import org.apache.druid.sql.calcite.expression.builtin.DateTruncOperatorConversion; +import org.apache.druid.sql.calcite.expression.builtin.ParseLongOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.RegexpExtractOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.StrposOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.TimeExtractOperatorConversion; @@ -83,6 +84,8 @@ public class ExpressionsTest extends CalciteTestBase .add("y", ValueType.LONG) .add("z", ValueType.FLOAT) .add("s", ValueType.STRING) + .add("hexstr", ValueType.STRING) + .add("intstr", ValueType.STRING) .add("spacey", ValueType.STRING) .add("tstr", ValueType.STRING) .add("dstr", ValueType.STRING) @@ -95,6 +98,8 @@ public class ExpressionsTest extends CalciteTestBase .put("y", 3.0) .put("z", -2.25) .put("s", "foo") + .put("hexstr", "EF") + .put("intstr", "-100") .put("spacey", " hey there ") .put("tstr", "2000-02-03 04:05:06") .put("dstr", "2000-02-03") @@ -198,6 +203,52 @@ public class ExpressionsTest extends CalciteTestBase ); } + @Test + public void testParseLong() + { + testExpression( + rexBuilder.makeCall( + new ParseLongOperatorConversion().calciteOperator(), + inputRef("intstr") + ), + DruidExpression.fromExpression("parse_long(\"intstr\")"), + -100L + ); + + testExpression( + rexBuilder.makeCall( + new ParseLongOperatorConversion().calciteOperator(), + inputRef("hexstr"), + rexBuilder.makeExactLiteral(BigDecimal.valueOf(16)) + ), + DruidExpression.fromExpression("parse_long(\"hexstr\",16)"), + 239L + ); + + testExpression( + rexBuilder.makeCall( + new ParseLongOperatorConversion().calciteOperator(), + rexBuilder.makeCall( + SqlStdOperatorTable.CONCAT, + rexBuilder.makeLiteral("0x"), + inputRef("hexstr") + ), + rexBuilder.makeExactLiteral(BigDecimal.valueOf(16)) + ), + DruidExpression.fromExpression("parse_long(concat('0x',\"hexstr\"),16)"), + 239L + ); + + testExpression( + rexBuilder.makeCall( + new ParseLongOperatorConversion().calciteOperator(), + inputRef("hexstr") + ), + DruidExpression.fromExpression("parse_long(\"hexstr\")"), + NullHandling.sqlCompatible() ? null : 0L + ); + } + @Test public void testPosition() {